Fixed time handling issues and missing map street names. (#431)

# Fixes
- Fixed `FileIndex` loading if time hint is specified but start/stop not specified
- Fixed `DataLoader` handling of messages containing both P1 and system timestamps
- Fixed dtype on `MeasurementDetails` GPS and system time numpy arrays
- Fixed missing street names on `p1_display` map

Author: adamshapiro0

python/fusion_engine_client/analysis/analyzer.py

@@ -92,29 +92,30 @@ def _build_map_style(mapbox_token: Optional[str]):
     """!
     @brief Build a `layout.map.style` value for a MapLibre-based Scattermap figure.
 
-    If a Mapbox access token is available, pull Mapbox satellite tiles via a custom raster style spec (the mechanism
-    MapLibre-based maps use in place of the old `layout.mapbox.accesstoken` field, which no longer exists). Otherwise,
-    fall back to Plotly's built-in token-free `satellite-streets` style, which serves ESRI World Imagery aerial tiles
-    (max zoom 16, lower resolution than Mapbox) with OpenMapTiles street labels drawn on top.
+    If a Mapbox access token is available, pull Mapbox's rendered `satellite-streets-v12` tiles (imagery with street
+    labels composited on top) via a custom raster style spec, the mechanism MapLibre-based maps use in place of the
+    old `layout.mapbox.accesstoken` field, which no longer exists. Otherwise, fall back to Plotly's built-in
+    token-free `satellite-streets` style, which serves ESRI World Imagery aerial tiles (max zoom 16, lower resolution
+    than Mapbox) with OpenMapTiles street labels drawn on top.
     """
     if not mapbox_token:
         return 'satellite-streets'
 
     return {
         'version': 8,
         'sources': {
-            'mapbox-satellite': {
+            'mapbox-satellite-streets': {
                 'type': 'raster',
                 'tiles': [
-                    f'https://api.mapbox.com/v4/mapbox.satellite/{{z}}/{{x}}/{{y}}@2x.jpg90'
+                    f'https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v12/tiles/256/{{z}}/{{x}}/{{y}}@2x'
                     f'?access_token={mapbox_token}'
                 ],
                 'tileSize': 256,
                 'attribution': '© Mapbox',
             },
         },
         'layers': [
-            {'id': 'mapbox-satellite-layer', 'type': 'raster', 'source': 'mapbox-satellite'},
+            {'id': 'mapbox-satellite-streets-layer', 'type': 'raster', 'source': 'mapbox-satellite-streets'},
         ],
     }
 

python/fusion_engine_client/analysis/data_loader.py

@@ -441,8 +441,12 @@ def _read(self,
         # fast reading, messages with system times may have their index entry timestamps set to NAN since A) they can
         # occur in a log before P1 time is established, and B) there's not necessarily a direct way to convert between
         # system and P1 time.
-        p1_time_messages_requested = any([t in messages_with_p1_time for t in needed_message_types])
-        system_time_messages_requested = any([t in messages_with_system_time for t in needed_message_types])
+        requested_messages_with_p1_time = any([t in messages_with_p1_time for t in needed_message_types])
+        requested_messages_with_system_time = any([t in messages_with_system_time for t in needed_message_types])
+        all_system_time_messages_have_p1_time = all([t in messages_with_p1_time
+                                                     for t in needed_message_types if t in messages_with_system_time])
+        requested_messages_with_only_system_time = (requested_messages_with_system_time
+                                                    and not all_system_time_messages_have_p1_time)
 
         # Create a dict with references to the requested types only to be returned below. If any data was already
         # cached, it will be present in self.data and populated here.
@@ -471,8 +475,8 @@ def _read(self,
 
         # If we need to establish t0 (either P1 time or system time), we will wait to apply the user's filter criteria.
         # We can get t0 from any message type.
-        need_t0 = self._need_t0 and p1_time_messages_requested
-        need_system_t0 = self._need_system_t0 and system_time_messages_requested
+        need_t0 = self._need_t0 and requested_messages_with_p1_time
+        need_system_t0 = self._need_system_t0 and requested_messages_with_system_time
 
         reader_max_messages_applied = False
         if need_t0 or need_system_t0:
@@ -486,7 +490,7 @@ def _read(self,
             self.reader.filter_in_place(None, source_ids=source_ids)
 
             # If the user is requiring (valid) P1 timestamps, filter to those now.
-            if require_p1_time and not system_time_messages_requested:
+            if require_p1_time and not requested_messages_with_only_system_time:
                 self.reader.filter_out_invalid_p1_times()
 
             # If the user requested max messages, tell the reader to return max N results. The reader only supports this
@@ -497,7 +501,7 @@ def _read(self,
             # not system time. The read_next() call below will apply this condition and only return messages with valid
             # system time.
             if (max_messages is not None and self.reader.have_index() and
-                    not (require_system_time and system_time_messages_requested)):
+                    not (require_system_time and requested_messages_with_system_time)):
                 reader_max_messages_applied = True
                 if max_messages >= 0:
                     self.reader.filter_in_place(slice(None, max_messages))

python/fusion_engine_client/messages/measurement_details.py

@@ -133,13 +133,13 @@ def to_numpy(cls, messages):
 
         idx = time_source == SystemTimeSource.GPS_TIME
         if np.any(idx):
-            gps_time = np.full_like(time_source, np.nan)
+            gps_time = np.full(time_source.shape, np.nan)
             gps_time[idx] = measurement_time[idx]
             result['gps_time'] = gps_time
 
         idx = time_source == SystemTimeSource.TIMESTAMPED_ON_RECEPTION
         if np.any(idx):
-            system_time = np.full_like(time_source, np.nan)
+            system_time = np.full(time_source.shape, np.nan)
             system_time[idx] = measurement_time[idx]
             result['system_time'] = system_time
 

python/fusion_engine_client/parsers/file_index.py

@@ -376,11 +376,11 @@ def get_time_range(self, start: Union[Timestamp, float] = None, stop: Union[Time
         if len(self._data) == 0:
             return FileIndex(data=np.copy(self._data), t0=self.t0)
         # No time bounds specified. Return the complete dataset.
-        elif start is None and stop is None:
+        elif start is None and stop is None and hint is None:
             return FileIndex(data=np.copy(self._data), t0=self.t0)
         # If there's no P1 timestamps in the index file whatsoever, t0 will be None. In that case, we cannot apply time
         # bounds to the data, since they are based on P1 time. This should be extremely rare.
-        elif self.t0 is None:
+        elif (start is not None or stop is not None) and self.t0 is None:
             raise IndexError(f'No P1 timestamps present in index. Cannot apply time bounds. '
                              f'[start={start}, stop={stop}]')
         else:

python/tests/test_file_index.py

@@ -353,9 +353,17 @@ def _lower_bound(time):
 
     # If the log does not contain P1 time, slicing it by time is not supported.
     with pytest.raises(IndexError):
-        sliced_index = index[1.0:]
-        sliced_index = index[TimeRange(start=2.0, absolute=True)]
-        sliced_index = index[TimeRange(start=2.0, absolute=False)]
+        index[1.0:]
+    with pytest.raises(IndexError):
+        index.get_time_range(start=1.0)
+    with pytest.raises(IndexError):
+        index[TimeRange(start=2.0, absolute=True)]
+    with pytest.raises(IndexError):
+        index[TimeRange(start=2.0, absolute=False)]
+
+    # However, if you don't set start or stop, setting hint should still work.
+    sliced_index = index.get_time_range(hint='include_nans')
+    assert (sliced_index.message_index == [e[3] for e in raw_data]).all()
 
 
 def test_empty_index():