fix detection estimation

pyproject.toml

@@ -26,7 +26,7 @@ matplotlib = "^3.5.2"
 scipy = "^1.4.1"
 gdown = "^4.7.1"
 ultralytics = "^8.0.148"
-notebook = "^6.5.5"
+notebook = "^7.0.6"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^7.4.0"
@@ -41,11 +41,11 @@ flake8 = "^5.0.4"
 pre-commit = "^3.2.2"
 
 # Notebook
-jupyterlab = "^3.6.2"
-ipywidgets = "^8.0.6"
-jupyterlab-lsp = "^4.0.1"
+jupyterlab = "^4.0.7"
+ipywidgets = "^8.1.1"
+jupyterlab-lsp = "^5.0.0"
 python-lsp-server = {extras = ["all"], version = "^1.7.1"}
-jupyterlab-vim = "^0.16.0"
+jupyterlab-vim = "^4.1.0"
 nbdime = "^3.2.1"
 altair = "^4.2.2"
 

spatialyze/video_processor/stages/detection_estimation/__init__.py

@@ -60,7 +60,7 @@ def _run(self, payload: "Payload"):
             return keep, {DetectionEstimation.classname(): [[]] * len(keep)}
 
         ego_views = get_ego_views(payload.video)
-        ego_views = [shapely.wkb.loads(view.to_ewkb(), hex=True) for view in ego_views]
+        # ego_views: list[shapely.geometry.Polygon] = [shapely.wkb.loads(view.to_ewkb(), hex=True) for view in ego_views]
 
         skipped_frame_num = []
         next_frame_num = 0
@@ -78,14 +78,17 @@ def _run(self, payload: "Payload"):
             if i != next_frame_num:
                 skipped_frame_num.append(i)
                 metadata.append([])
+                print('skip   ', i, next_frame_num)
                 continue
 
             next_frame_num = i + 1
 
             det, _, dids = dets[i]
+            print('--new    ', i, new_car(dets, i, i + 5))
             if new_car(dets, i, i + 5) <= i + 1:
                 # will not map segment if cannot skip in the first place
                 metadata.append([])
+                print('new    ', i, new_car(dets, i, i + 5))
                 continue
 
             start_detection_time = time.time()
@@ -97,13 +100,15 @@ def _run(self, payload: "Payload"):
                 (time.time() - start_detection_time, len(det), len(all_detection_info), times)
             )
 
-            all_detection_info_pruned, det = prune_detection(
-                all_detection_info, det, self.predicates
-            )
+            # all_detection_info_pruned, det = prune_detection(
+            #     all_detection_info, det, self.predicates
+            # )
+            all_detection_info_pruned = all_detection_info
 
             if len(det) == 0 or len(all_detection_info_pruned) == 0:
                 # skipped_frame_num.append(i)
                 metadata.append([])
+                print('0      ', i)
                 continue
 
             start_generate_sample_plan = time.time()
@@ -114,6 +119,7 @@ def _run(self, payload: "Payload"):
             next_frame_num = next_sample_plan.get_next_frame_num()
             next_frame_num = new_car(dets, i, next_frame_num)
             logger.info(f"founded next_frame_num {next_frame_num}")
+            print('next   ', i)
             metadata.append(all_detection_info)
 
             next_action_type = next_sample_plan.get_action_type()
@@ -158,7 +164,7 @@ def new_car(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"):
         future_det = dets[j][0]
         if len(future_det) > len_det:
             return j
-    return nxt
+    return min(nxt, len(dets) - 1)
 
 
 def objects_count_change(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"):
@@ -173,7 +179,7 @@ def objects_count_change(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"):
 
 
 def get_ego_views(video: "Video") -> "list[postgis.Polygon]":
-    indices, view_areas = get_views(video, distance=100, skip=False)
+    indices, view_areas = get_views(video, distance=100)
     views_raw = database.execute(
         sql.SQL(
             """
@@ -220,7 +226,7 @@ def generate_sample_plan_once(
     next_frame_num: "int",
     ego_views: "list[postgis.Polygon]",
     all_detection_info: "list[DetectionInfo] | None" = None,
-    fps: "int" = 13,
+    fps: "float" = 13,
 ) -> "tuple[SamplePlan, None]":
     assert all_detection_info is not None
     next_sample_plan = generate_sample_plan(
@@ -234,7 +240,7 @@ def construct_estimated_all_detection_info(
     detection_ids: "list[DetectionId]",
     ego_config: "CameraConfig",
     ego_trajectory: "list[trajectory_3d]",
-) -> "list[DetectionInfo]":
+) -> "tuple[list[DetectionInfo], float]":
     _times = time.time()
     all_detections = []
     for det, did in zip(detections, detection_ids):

spatialyze/video_processor/stages/detection_estimation/detection_estimation.py

@@ -152,7 +152,7 @@ class SamplePlan:
     next_frame_num: int
     all_detection_info: "list[DetectionInfo]"
     ego_views: "list[postgis.Polygon]"
-    fps: int = 12
+    fps: float = 12
     current_priority: "float | None" = None
     action: "Action | None" = None
 
@@ -259,15 +259,15 @@ def construct_all_detection_info(
     ego_config: "CameraConfig",
     ego_trajectory: "list[trajectory_3d]",
     all_detections: "list[obj_detection]",
-):
+) -> tuple[list[DetectionInfo], list[float]]:
     all_detection_info: "list[DetectionInfo]" = []
     if len(all_detections) == 0:
         return all_detection_info, []
 
     # ego_road_polygon_info = get_largest_polygon_containing_point(ego_config)
     detections_polygon_mapping, times = get_detection_polygon_mapping(all_detections, ego_config)
     if len(detections_polygon_mapping) == 0:
-        return [], times
+        return all_detection_info, times
 
     # assert len(all_detections) == len(detections_polygon_mapping)
     for detection in all_detections:
@@ -298,7 +298,7 @@ def generate_sample_plan(
     all_detection_info: "list[DetectionInfo]",
     ego_views: "list[postgis.Polygon]",
     view_distance: float,
-    fps: int = 12,
+    fps: float = 12,
 ):
     ### the object detection with higher priority doesn't necessarily get sampled first,
     # it also based on the sample plan

spatialyze/video_processor/stages/detection_estimation/segment_mapping.py

@@ -186,7 +186,7 @@ def get_detection_polygon_mapping(detections: "list[obj_detection]", ego_config:
     Given a list of detections, return a list of RoadSegmentWithHeading
     """
     # start_time = time.time()
-    times = []
+    times: list[float] = []
     times.append(time.time())
     results = map_detections_to_segments(detections, ego_config)
     times.append(time.time())
@@ -202,13 +202,13 @@ def get_detection_polygon_mapping(detections: "list[obj_detection]", ego_config:
     times.append(time.time())
     mapped_polygons = reformat_return_polygon(mapped_polygons)
     times.append(time.time())
+    mapped_road_polygon_info: "dict[DetectionId, RoadPolygonInfo]" = {}
     if any(p.road_type == "intersection" for p in mapped_polygons):
-        return {}, times
+        return mapped_road_polygon_info, times
     times.append(time.time())
     fov_lines = get_fov_lines(ego_config)
     times.append(time.time())
 
-    mapped_road_polygon_info: "dict[DetectionId, RoadPolygonInfo]" = {}
     for order_id, road_polygon in list(zip(order_ids, mapped_polygons)):
         frame_idx = detections[0].detection_id.frame_idx
         det_id = DetectionId(frame_idx=frame_idx, obj_order=order_id)
@@ -221,7 +221,7 @@ def get_detection_polygon_mapping(detections: "list[obj_detection]", ego_config:
 
         # assert all(isinstance(line, sg.LineString) for line in segmentlines)
 
-        p = swkb.loads(road_polygon.to_ewkb(), hex=True)
+        p = swkb.loads(roadpolygon.to_ewkb(), hex=True)
         assert isinstance(p, sg.Polygon)
         XYs: "Tuple[array.array[float], array.array[float]]" = p.exterior.xy
         assert isinstance(XYs, tuple)

spatialyze/video_processor/stages/detection_estimation/utils.py

@@ -317,13 +317,14 @@ def get_car_exits_view_frame_num(
     start_frame_num = detection_info.detection_id.frame_idx
     car_speed = max_car_speed(road_type)
     car_heading += 90
-    while detection_info.detection_id.frame_idx + 1 < max_frame_num:
-        next_frame_num = detection_info.detection_id.frame_idx + 1
+    frame_idx = detection_info.detection_id.frame_idx
+    while frame_idx + 1 < max_frame_num:
+        next_frame_num = frame_idx + 1
         next_ego_view = ego_views[next_frame_num]
         next_ego_view = shapely.wkb.loads(next_ego_view.to_ewkb(), hex=True)
         duration = (next_frame_num - start_frame_num) / fps
         next_car_loc = car_move(car_loc, car_heading, car_speed, duration)
         if not next_ego_view.contains(shapely.geometry.Point(next_car_loc[:2])):
-            return max(detection_info.detection_id.frame_idx, start_frame_num + 1)
-        detection_info.detection_id.frame_idx = next_frame_num
+            return max(frame_idx, start_frame_num + 1)
+        frame_idx = next_frame_num
     return max_frame_num

spatialyze/video_processor/stream/exit_frame_sampler.py

@@ -36,7 +36,7 @@ def _stream(self, video: Video):
             return self.detections.stream(video)
 
         ego_views = get_ego_views(video)
-        ego_views = [shapely.wkb.loads(view.to_ewkb(), hex=True) for view in ego_views]
+        # ego_views = [shapely.wkb.loads(view.to_ewkb(), hex=True) for view in ego_views]
 
         skipped_frame_num = []
         next_frame_num = 0
@@ -51,17 +51,16 @@ def _stream(self, video: Video):
                 yield True
                 continue
 
-            if i != next_frame_num:
-                yield skip
+            if i != next_frame_num or isinstance(detection, Skip):
+                yield False
                 continue
 
             next_frame_num = i + 1
 
             det, _, dids = detection
-            if new_car(detections, 5) <= i + 1:
+            if new_car(detections, min(5, len(video) - i - 1)) <= 1:
                 # do not map segment if cannot skip in the first place
-                skipped_frame_num.append(i)
-                yield skip
+                yield True
                 continue
 
             start_detection_time = time.time()
@@ -79,9 +78,7 @@ def _stream(self, video: Video):
             all_detection_info_pruned = all_detection_info
 
             if len(det) == 0 or len(all_detection_info_pruned) == 0:
-                skipped_frame_num.append(i)
-                # metadata.append([])
-                yield skip
+                yield True
                 continue
 
             start_generate_sample_plan = time.time()
@@ -90,9 +87,9 @@ def _stream(self, video: Video):
             )
             total_sample_plan_time.append(time.time() - start_generate_sample_plan)
             next_frame_num = next_sample_plan.get_next_frame_num()
-            next_frame_num = new_car(detections, next_frame_num - i)
+            next_frame_num = i + new_car(detections, min(next_frame_num, len(video) - 1) - i)
             logger.info(f"founded next_frame_num {next_frame_num}")
-            yield detection
+            yield True
 
             next_action_type = next_sample_plan.get_action_type()
             if next_action_type not in action_type_counts:
@@ -128,7 +125,7 @@ class FutureIterator(Generic[T], Iterator[T]):
     def __init__(self, it: Iterable[T]):
         self.it = iter(it)
         self.idx = -1
-        self.mem = []
+        self.mem: list[T | None] = []
 
     def __next__(self):
         if self.idx >= 0:
@@ -144,13 +141,19 @@ def __getitem__(self, item: int):
                 self.mem.append(next(self.it))
             return self.mem[self.idx + item]
         except StopIteration:
+            if item == 0:
+                raise StopIteration
             return None
 
 
 def new_car(detections: FutureIterator[Detection3D | Skip], nxt: int):
-    len_det = len(detections[0][0])
+    detection = detections[0]
+    assert detection is not None
+    assert not isinstance(detection, Skip)
+    len_det = len(detection[0])
     for i in range(1, nxt + 1):
         det = detections[i]
+        assert not isinstance(det, Skip)
         if det is None:
             return i - 1
         future_det = det[0]

spatialyze/video_processor/stream/object_type_pruner.py

@@ -1,7 +1,7 @@
 from ...predicate import PredicateNode
 from ..stages.detection_2d.object_type_filter import ObjectTypeFilter
 from ..video import Video
-from .data_types import Detection2D, Skip
+from .data_types import Detection2D, Skip, skip
 from .stream import Stream
 
 
@@ -21,7 +21,7 @@ def _stream(self, video: Video):
 
         for detection_2d in self.detections.stream(video):
             if isinstance(detection_2d, Skip):
-                yield Skip()
+                yield skip
                 continue
 
             det, class_mapping, ids = detection_2d