Fix InView Typing (#17)

* update tracking3d

* style: [CI] format

* update ablation

* style: [CI] format

* update type

* fix type

* fix types

* do not change environment name

* fix typing

* fix type

* style: [CI] format

* clean up

* fix lint

* fix type error

* style: [CI] format

* fix typo

* update

* style: [CI] format

* .

* style: [CI] format

---------

Co-authored-by: Github Actions Bot <spatialyze-actions-bot@users.noreply.github.com>

playground/run-ablation.ipynb

@@ -172,6 +172,7 @@
     "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_road import FromTracking2DAndRoad\n",
     "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_depth import FromTracking2DAndDepth\n",
     "from spatialyze.video_processor.stages.tracking_3d.tracking_3d import Tracking3DResult, Tracking3D\n",
+    "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_detection_3d import FromTracking2DAndDetection3D as FromT2DAndD3D\n",
     "\n",
     "from spatialyze.video_processor.stages.segment_trajectory import SegmentTrajectory\n",
     "from spatialyze.video_processor.stages.segment_trajectory.construct_segment_trajectory import SegmentPoint\n",
@@ -362,7 +363,7 @@
     "    # names = set(sampled_scenes)\n",
     "    filtered_videos = [\n",
     "        n for n in videos\n",
-    "        if n[6:10] in names # and 'FRONT' in n # and n.endswith('FRONT')\n",
+    "        if n[6:10] in names and 'FRONT' in n # and n.endswith('FRONT')\n",
     "    ]\n",
     "    N = len(filtered_videos)\n",
     "    print('# of filtered videos:', N)\n",
@@ -420,14 +421,14 @@
     "\n",
     "                # Ingest Trackings\n",
     "                ego_meta = frames.interpolated_frames\n",
-    "                sortmeta = FromTracking2DAndRoad.get(output)\n",
+    "                sortmeta = Tracking3D.get(output)\n",
     "                assert sortmeta is not None\n",
     "                segment_trajectories = FromTracking3D.get(output)\n",
     "                tracks = get_tracks(sortmeta, ego_meta, segment_trajectories)\n",
     "                for obj_id, track in tracks.items():\n",
     "                    trajectory = format_trajectory(name, obj_id, track)\n",
     "                    if trajectory:\n",
-    "                        insert_trajectory(database, *trajectory)\n",
+    "                        insert_trajectory(database, *trajectory[0])\n",
     "\n",
     "                # Ingest Camera\n",
     "                accs: 'ACameraConfig' = []\n",
@@ -569,9 +570,6 @@
     "\n",
     "    # Object Filter\n",
     "    if object_filter:\n",
-    "        # if isinstance(object_filter, bool):\n",
-    "        #     object_filter = ['car', 'truck']\n",
-    "        # TODO: filter objects based on predicate\n",
     "        pipeline.add_filter(ObjectTypeFilter(predicate=predicate))\n",
     "\n",
     "    # 3D Detection\n",
@@ -592,10 +590,11 @@
     "        cache=ss_cache,\n",
     "    ))\n",
     "\n",
-    "    if geo_depth:\n",
-    "        pipeline.add_filter(FromTracking2DAndRoad())\n",
-    "    else:\n",
-    "        pipeline.add_filter(FromTracking3DAndDepth())\n",
+    "    pipeline.add_filter(FromT2DAndD3D())\n",
+    "    # if geo_depth:\n",
+    "    #     pipeline.add_filter(FromTracking2DAndRoad())\n",
+    "    # else:\n",
+    "    #     pipeline.add_filter(FromTracking2DAndDepth())\n",
     "\n",
     "    # Segment Trajectory\n",
     "    # pipeline.add_filter(FromTracking3D())\n",
@@ -902,9 +901,8 @@
    },
    "outputs": [],
    "source": [
-    "tests = ['de', 'optde', 'noopt', 'inview', 'objectfilter', 'geo', 'opt']\n",
-    "# tests = ['de', 'optde']\n",
-    "# tests = ['de']\n",
+    "tests = ['optde', 'de', 'noopt', 'inview', 'objectfilter', 'geo', 'opt']\n",
+    "tests = ['de', 'noopt', 'inview', 'objectfilter']\n",
     "# random.shuffle(tests)\n",
     "\n",
     "for _test in tests:\n",

playground/run-all-local

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+tmux new-session -d -s run-test -n run-test-local || echo 'hi'
+tmux send-keys -t run-test-local 'docker container start mobilitydb' Enter
+tmux send-keys -t run-test-local 'cd ~/Documents/spatialyze' Enter
+tmux send-keys -t run-test-local 'rm -rf ./outputs/run/*' Enter
+tmux send-keys -t run-test-local 'rm -rf ./run-ablation.py' Enter
+tmux send-keys -t run-test-local 'python ./spatialyze/utils/ingest_road.py "./data/scenic/road-network/boston-seaport"' Enter
+tmux send-keys -t run-test-local 'jupyter nbconvert --to python ./playground/run-ablation.ipynb && mv playground/run-ablation.py .' Enter
+tmux send-keys -t run-test-local 'python run-ablation.py' Enter

pyproject.toml

@@ -105,10 +105,10 @@ exclude = 'spatialyze/video_processor/modules'
 pythonVersion = '3.10'
 ignore = [
   'spatialyze/video_processor/modules',
+  'spatialyze/video_processor/stages/detection_2d/ground_truth.py',
   'spatialyze/video_processor/stages/detection_estimation',
-  'spatialyze/video_processor/stages/segment_trajectory',
+  'spatialyze/video_processor/stages/segment_trajectory/construct_segment_trajectory.py',
   'spatialyze/video_processor/utils/process_pipeline.py',
   'spatialyze/video_processor/utils/query_analyzer.py',
   'spatialyze/video_processor/utils/preprocess.py',
-  'spatialyze/video_processor'
 ]

spatialyze/video_processor/stages/detection_2d/object_type_filter.py

@@ -23,10 +23,11 @@
 
 class ObjectTypeFilter(Detection2D):
     def __init__(self, types: "list[str] | None" = None, predicate: "PredicateNode | None" = None):
-        assert (types is None) != (predicate is None), "Can only except either types or predicate"
         if types is None:
+            assert predicate is not None, "Can only except either types or predicate"
             self.types = list(FindType()(predicate))
         else:
+            assert types is not None, "Can only except either types or predicate"
             self.types = types
         print("types", self.types)
 

spatialyze/video_processor/stages/detection_estimation/segment_mapping.py

@@ -137,13 +137,13 @@ class RoadPolygonInfo:
     segment_headings: "list[float]"
     contains_ego: bool
     ego_config: "CameraConfig"
-    fov_lines: "Tuple[Float22, Float22]"
+    fov_lines: "tuple[Float22, Float22]"
 
     def __post_init__(self):
         if len(self.segment_lines) == 0:
             return
 
-        start_segment_map: "dict[Float2, Tuple[shapely.geometry.LineString, float]]" = {}
+        start_segment_map: "dict[Float2, tuple[shapely.geometry.LineString, float]]" = {}
         ends: "set[Float2]" = set()
         for line, heading in zip(self.segment_lines, self.segment_headings):
             start = line.coords[0]

spatialyze/video_processor/stages/detection_estimation/utils.py

@@ -12,7 +12,8 @@
 if TYPE_CHECKING:
     from ...camera_config import CameraConfig
     from .detection_estimation import DetectionInfo
-    from .segment_mapping import CameraPolygonMapping, RoadPolygonInfo
+    from .optimized_segment_mapping import RoadPolygonInfo
+    from .segment_mapping import CameraPolygonMapping
 
 
 logger = logging.getLogger(__name__)

spatialyze/video_processor/stages/in_view/in_view.py

@@ -48,16 +48,17 @@ def __init__(
     ):
         super().__init__()
         self.distance = distance
-        assert (roadtypes is None) != (
-            predicate is None
-        ), "Can only except either segment_type or predicate"
-        self.roadtypes: "list[str] | None" = None
-        self.predicate: "PredicateNode | None" = None
+        assert (
+            roadtypes is not None or predicate is not None
+        ), "At least one of roadtypes or predicate must be specified"
         if roadtypes is not None:
+            assert predicate is None, "Can only except either segment_type or predicate"
             self.roadtypes = roadtypes if isinstance(roadtypes, list) else [roadtypes]
-        if predicate is not None:
+            self.predicate = None
+        elif predicate is not None:
+            assert roadtypes is None, "Can only except either segment_type or predicate"
             self.roadtypes, self.predicate_str = create_inview_predicate(predicate)
-            self.predicate = eval(self.predicate_str)
+            self.predicate = eval(str(self.predicate_str))
 
     def __repr__(self) -> str:
         return f"InView(distance={self.distance}, roadtype={self.roadtypes}, predicate={self.predicate_str})"
@@ -190,7 +191,7 @@ def get_views(payload: "Payload", distance: "float" = 100, skip: "bool" = True):
         )
     )
 
-    _extrinsics = np.stack(extrinsics)
+    _extrinsics = np.stack(extrinsics, dtype=np.floating)
     assert _extrinsics.shape == (N, 3, 4), _extrinsics.shape
 
     # convert 4 corner points from pixel-coordinate to world-coordinate
@@ -248,9 +249,13 @@ def visit_BoolOpNode(self, node: BoolOpNode):
             # print('annihilated', visited)
             return lit(annihilator)
         if all(isinstance(e, LiteralNode) for e in visited.exprs):
-            assert len({e.value for e in visited.exprs}) == 1, visited.exprs
+            exprs = visited.exprs
+            assert all(isinstance(e, LiteralNode) for e in exprs), exprs
+            assert len({e.value for e in exprs if isinstance(e, LiteralNode)}) == 1, visited.exprs
             # print('all', visited)
-            return lit(visited.exprs[0].value)
+            e = exprs[0]
+            assert isinstance(e, LiteralNode), e
+            return lit(e.value)
         exprs = [e for e in visited.exprs if not isinstance(e, LiteralNode)]
         return BoolOpNode(visited.op, exprs) if len(exprs) > 1 else exprs[0]
 
@@ -294,7 +299,7 @@ def visit_CallNode(self, node: CallNode):
             else:
                 assert isinstance(segment, CallNode), segment
                 assert segment == F.same_region().fn, segment
-                assert len(segment.params) == 1, segment.param
+                assert len(segment.params) == 1, segment.params
                 assert isinstance(segment.params[0], LiteralNode), segment.params[0]
                 assert isinstance(segment.params[0].value, str), segment.params[0]
                 return F.is_roadtype(segment.params[0])
@@ -405,20 +410,24 @@ def visit_BoolOpNode(self, node: BoolOpNode):
                 return F.ignore_roadtype()
 
         # Boolean Absorption
-        is_roadtypes = [
-            e.params[0].value.lower()
-            for e in _exprs
-            if isinstance(e, CallNode) and e.fn == IS_ROADTYPE
+        e_params = [e.params[0] for e in _exprs if isinstance(e, CallNode) and e.fn == IS_ROADTYPE]
+        assert all(
+            isinstance(e, LiteralNode) and isinstance(e.value, str) for e in e_params
+        ), e_params
+        _is_roadtypes = [
+            v.lower()
+            for v in (e.value for e in e_params if isinstance(e, LiteralNode))
+            if isinstance(v, str)
         ]
         nested_exprs = [e for e in _exprs if isinstance(e, BoolOpNode) and e.op != node.op]
-        assert len(is_roadtypes) + len(nested_exprs) == len(_exprs), (
-            len(is_roadtypes),
+        assert len(_is_roadtypes) + len(nested_exprs) == len(_exprs), (
+            len(_is_roadtypes),
             len(nested_exprs),
             len(_exprs),
             _exprs,
         )
 
-        is_roadtypes: "set[str]" = set(is_roadtypes)
+        is_roadtypes: "set[str]" = set(_is_roadtypes)
 
         def is_absorbed(e: "CallNode | BoolOpNode"):
             if isinstance(e, CallNode):
@@ -432,11 +441,25 @@ def is_absorbed(e: "CallNode | BoolOpNode"):
             else:
                 assert e.op == node.op, (node.op, e.op)
                 all_roadtype = all(isinstance(e, CallNode) and e.fn == IS_ROADTYPE for e in e.exprs)
-                return all_roadtype and {ee.params[0].value.lower() for ee in e.exprs}.issubset(
-                    is_roadtypes
-                )
-
-        nested_exprs = [e for e in nested_exprs if not any(map(is_absorbed, e.exprs))]
+                if not all_roadtype:
+                    return False
+                exprs = [ee.params[0] for ee in e.exprs if isinstance(ee, CallNode)]
+                assert all(
+                    isinstance(e, LiteralNode) and isinstance(e.value, str) for e in exprs
+                ), exprs
+                return {
+                    v.lower()
+                    for v in (ee.value for ee in exprs if isinstance(ee, LiteralNode))
+                    if isinstance(v, str)
+                }.issubset(is_roadtypes)
+
+        def all_not_absorbed(e: "BoolOpNode"):
+            exprs = e.exprs
+            assert all(isinstance(e, (CallNode, BoolOpNode)) for e in exprs), exprs
+            exprs = (e for e in exprs if isinstance(e, (CallNode, BoolOpNode)))
+            return not any(map(is_absorbed, exprs))
+
+        nested_exprs = filter(all_not_absorbed, nested_exprs)
 
         _exprs = [*map(F.is_roadtype, sorted(is_roadtypes)), *nested_exprs]
         if len(_exprs) == 1:

spatialyze/video_processor/stages/segment_trajectory/__init__.py

@@ -10,7 +10,13 @@
 from ..detection_estimation.utils import trajectory_3d
 from ..stage import Stage
 from ..tracking_2d.strongsort import StrongSORT
-from .construct_segment_trajectory import SegmentPoint, calibrate
+from .construct_segment_trajectory import (
+    InvalidSegmentPoint,
+    PolygonAndId,
+    SegmentPoint,
+    ValidSegmentPoint,
+    calibrate,
+)
 
 SegmentTrajectoryMetadatum = Dict[int, SegmentPoint]
 
@@ -51,7 +57,7 @@ def _run(self, payload: "Payload"):
 
     @classmethod
     def encode_json(cls, o: "Any"):
-        if isinstance(o, SegmentPoint):
+        if isinstance(o, ValidSegmentPoint):
             return {
                 "detection_id": tuple(o.detection_id),
                 "car_loc3d": o.car_loc3d,
@@ -65,6 +71,16 @@ def encode_json(cls, o: "Any"):
                 "next": None if o.next is None else tuple(o.next.detection_id),
                 "prev": None if o.prev is None else tuple(o.prev.detection_id),
             }
+        if isinstance(o, InvalidSegmentPoint):
+            return {
+                "detection_id": tuple(o.detection_id),
+                "car_loc3d": o.car_loc3d,
+                "timestamp": str(o.timestamp),
+                "obj_id": o.obj_id,
+                "type": o.type,
+                "next": None if o.next is None else tuple(o.next.detection_id),
+                "prev": None if o.prev is None else tuple(o.prev.detection_id),
+            }
         if isinstance(o, (RoadPolygonInfo, RoadPolygonInfo_)):
             return {
                 "id": o.id,
@@ -78,6 +94,12 @@ def encode_json(cls, o: "Any"):
                 "fov_lines": o.fov_lines,
             }
 
+        if isinstance(o, PolygonAndId):
+            return {
+                "id": o.id,
+                "polygon": str(o.polygon),
+            }
+
 
 def construct_trajectory(source: "Payload"):
     obj_3d_trajectories: "dict[int, list[Tuple[trajectory_3d, DetectionInfo, int]]]" = {}