From b2285635e08cf0abb0466fd6f584ebc6e1477f7c Mon Sep 17 00:00:00 2001 From: "Chanwut (Mick) Kittivorawong" Date: Fri, 3 Nov 2023 03:54:46 -0700 Subject: [PATCH] generate videos --- evaluation/ablation/longvideo.ipynb | 253 + evaluation/ablation/test-stream-2.ipynb | 15419 ++++++++++++++++++++++ longvideo.py | 122 + 3 files changed, 15794 insertions(+) create mode 100644 evaluation/ablation/longvideo.ipynb create mode 100644 evaluation/ablation/test-stream-2.ipynb create mode 100644 longvideo.py diff --git a/evaluation/ablation/longvideo.ipynb b/evaluation/ablation/longvideo.ipynb new file mode 100644 index 0000000..9cbf217 --- /dev/null +++ b/evaluation/ablation/longvideo.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 21, + "id": "7f78c96d-1a45-4ced-8c65-d7c775992139", + "metadata": {}, + "outputs": [], + "source": [ + "import cv2\n", + "import os\n", + "import json\n", + "import pickle\n", + "from tqdm import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7267f20a-3b2f-4173-b613-ae36a0cd2505", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b2a726fa-f53a-4f86-bd1a-102f8371f6e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/data/chanwutk/projects/spatialyze-ablation\n" + ] + } + ], + "source": [ + "\n", + "def is_notebook() -> bool:\n", + " try:\n", + " shell = get_ipython().__class__.__name__\n", + " if shell == 'ZMQInteractiveShell':\n", + " # Jupyter notebook or qtconsole\n", + " return True\n", + " elif shell == 'TerminalInteractiveShell':\n", + " # Terminal running IPython\n", + " return False\n", + " else:\n", + " # Other type (?)\n", + " return False\n", + " except NameError:\n", + " # Probably standard Python interpreter\n", + " return False\n", + "\n", + "\n", + "if is_notebook():\n", + " %cd ../..\n", + " from tqdm.notebook import tqdm\n", + " from nbutils.report_progress import report_progress\n", + "else:\n", + " from tqdm import tqdm\n", + " from evaluation.ablation.nbutils.report_progress import report_progress" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "fa64ff96-ad9a-482f-8c17-8db1ffce043f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "/data/apperception-data/processed/nuscenes/full-dataset-v1.0/Trainval\n" + ] + } + ], + "source": [ + "NUSCENES_PROCESSED_DATA = \"NUSCENES_PROCESSED_DATA\"\n", + "print(NUSCENES_PROCESSED_DATA in os.environ)\n", + "print(os.environ['NUSCENES_PROCESSED_DATA'])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "ebc0dbe2-8236-4ed9-a048-3d64d456a094", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'scene-0001-CAM_FRONT'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "DATA_DIR = os.environ[NUSCENES_PROCESSED_DATA]\n", + "# with open(os.path.join(DATA_DIR, \"videos\", \"frames.pkl\"), \"rb\") as f:\n", + "# videos = pickle.load(f)\n", + "with open(os.path.join(DATA_DIR, 'videos', 'videos.json'), 'r') as f:\n", + " videos = json.load(f)\n", + "type(videos)\n", + "videos[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "fc2bbf71-3aa0-4d2a-a9b5-a6d27097a043", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0523 0778 467\n" + ] + } + ], + "source": [ + "\n", + "with open('./data/evaluation/video-samples/boston-seaport.txt', 'r') as f:\n", + " sampled_scenes = f.read().split('\\n')\n", + "print(sampled_scenes[0], sampled_scenes[-1], len(sampled_scenes))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c8a069a6-0bdc-4bc4-afc6-b96132306775", + "metadata": {}, + "outputs": [], + "source": [ + "with open(os.path.join(DATA_DIR, 'videos', 'boston-seaport-scene-' + sampled_scenes[0] + '-CAM_FRONT.pkl'), 'rb') as f:\n", + " video = pickle.load(f)\n", + "video_filename = video['filename']" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "03af58b4-db6d-40d4-b3ab-b9429d73056d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'boston-seaport-scene-0523-CAM_FRONT.mp4'" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "video_filename" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "38f3f36b-e381-4de7-81f9-fdc70810c271", + "metadata": {}, + "outputs": [], + "source": [ + "frames = []\n", + "cap = cv2.VideoCapture(os.path.join(DATA_DIR, 'videos', video_filename))\n", + "while cap.isOpened():\n", + " ret, frame = cap.read()\n", + " if not ret:\n", + " break\n", + " frames.append(frame)\n", + "cap.release()\n", + "cv2.destroyAllWindows()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "5f1b5b82-9ee4-459e-86f3-1fc863ae9502", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 2%|█▊ | 23/1000 [01:08<48:42, 2.99s/it]\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[28], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1000\u001b[39m)):\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m frame \u001b[38;5;129;01min\u001b[39;00m frames:\n\u001b[0;32m----> 6\u001b[0m \u001b[43mresult\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mframe\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 7\u001b[0m result\u001b[38;5;241m.\u001b[39mrelease() \n\u001b[1;32m 8\u001b[0m cv2\u001b[38;5;241m.\u001b[39mdestroyAllWindows()\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "result = cv2.VideoWriter('./loop-video.avi', \n", + " cv2.VideoWriter_fourcc(*'MJPG'), \n", + " 12, (1600, 900)) \n", + "for i in tqdm(range(2000)):\n", + " for frame in frames:\n", + " result.write(frame) \n", + "result.release() \n", + "cv2.destroyAllWindows()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9e4c5eb-7121-41a0-88ea-a742666695e6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/evaluation/ablation/test-stream-2.ipynb b/evaluation/ablation/test-stream-2.ipynb new file mode 100644 index 0000000..51c832f --- /dev/null +++ b/evaluation/ablation/test-stream-2.ipynb @@ -0,0 +1,15419 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "569e4726-2856-4e5f-a220-e3bef1c110e1", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Oct 31 17:24:04 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 530.30.02 Driver Version: 530.30.02 CUDA Version: 12.1 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA TITAN Xp On | 00000000:1A:00.0 Off | N/A |\n", + "| 31% 53C P2 61W / 250W| 3056MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + "| 1 NVIDIA TITAN Xp On | 00000000:1B:00.0 Off | N/A |\n", + "| 23% 33C P8 9W / 250W| 4MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + "| 2 NVIDIA TITAN Xp On | 00000000:3D:00.0 Off | N/A |\n", + "| 23% 27C P8 9W / 250W| 4MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + "| 3 NVIDIA TITAN Xp On | 00000000:3E:00.0 Off | N/A |\n", + "| 23% 35C P8 10W / 250W| 4MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + "| 4 NVIDIA TITAN Xp On | 00000000:88:00.0 Off | N/A |\n", + "| 23% 28C P8 9W / 250W| 4MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + "| 5 NVIDIA TITAN Xp On | 00000000:89:00.0 Off | N/A |\n", + "| 23% 29C P8 9W / 250W| 4MiB / 12288MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| 0 N/A N/A 5846 C ...nvs/spatialyze-benchmark/bin/python 1140MiB |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "import subprocess\n", + "import json\n", + "import os\n", + "import pickle\n", + "import traceback\n", + "import shutil\n", + "import socket\n", + "import time\n", + "import random\n", + "import math\n", + "\n", + "\n", + "from os import environ\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import psycopg2\n", + "\n", + "subprocess.Popen('nvidia-smi', shell=True).wait()\n", + "process = subprocess.Popen('docker container start mobilitydb_chanwutk', shell=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7bcaf7b9-8b3f-4f2f-ad00-08b197795820", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test freddie\n" + ] + } + ], + "source": [ + "hostname = socket.gethostname()\n", + "test = hostname.split(\"-\")[-1]\n", + "print(\"test\", test)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "df967c40-baae-4473-8ab4-0ddfd630eb59", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/data/chanwutk/projects/spatialyze-ablation\n" + ] + } + ], + "source": [ + "def is_notebook() -> bool:\n", + " try:\n", + " shell = get_ipython().__class__.__name__\n", + " if shell == 'ZMQInteractiveShell':\n", + " # Jupyter notebook or qtconsole\n", + " return True\n", + " elif shell == 'TerminalInteractiveShell':\n", + " # Terminal running IPython\n", + " return False\n", + " else:\n", + " # Other type (?)\n", + " return False\n", + " except NameError:\n", + " # Probably standard Python interpreter\n", + " return False\n", + "\n", + "\n", + "if is_notebook():\n", + " %cd ../..\n", + " from tqdm.notebook import tqdm\n", + " from nbutils.report_progress import report_progress\n", + "else:\n", + " from tqdm import tqdm\n", + " from evaluation.ablation.nbutils.report_progress import report_progress" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "edb4d993-d0a4-49d5-8b4d-1b3a62f66da4", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mobilitydb_chanwutk\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "process.wait()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ddcd5926-911d-437d-b67f-6814d5eb7c0b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CUDA is available.\n", + " > 0: NVIDIA TITAN Xp\n", + " 1: NVIDIA TITAN Xp\n", + " 2: NVIDIA TITAN Xp\n", + " 3: NVIDIA TITAN Xp\n", + " 4: NVIDIA TITAN Xp\n" + ] + } + ], + "source": [ + "from spatialyze.video_processor.camera_config import camera_config\n", + "from spatialyze.video_processor.payload import Payload\n", + "from spatialyze.video_processor.pipeline import Pipeline\n", + "from spatialyze.video_processor.video import Video\n", + "from spatialyze.video_processor.metadata_json_encoder import MetadataJSONEncoder" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "863b7bd6-7bc4-4658-bb06-043ba955aef3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Stages\n", + "from spatialyze.video_processor.stages.in_view import InView\n", + "\n", + "from spatialyze.video_processor.stages.decode_frame.decode_frame import DecodeFrame as StageDecodeFrame\n", + "from spatialyze.video_processor.stages.decode_frame.parallel_decode_frame import ParallelDecodeFrame\n", + "\n", + "from spatialyze.video_processor.stages.detection_2d.detection_2d import Detection2D\n", + "from spatialyze.video_processor.stages.detection_2d.yolo_detection import YoloDetection\n", + "from spatialyze.video_processor.stages.detection_2d.object_type_filter import ObjectTypeFilter\n", + "from spatialyze.video_processor.stages.detection_2d.ground_truth import GroundTruthDetection" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6ee9880e-27d0-47f3-a3dd-36b70d199d8a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from spatialyze.video_processor.stages.detection_3d.from_detection_2d_and_road import FromDetection2DAndRoad as StageFromDetection2DAndRoad\n", + "from spatialyze.video_processor.stages.detection_3d.from_detection_2d_and_depth import FromDetection2DAndDepth as StageFromDetection2DAndDepth\n", + "\n", + "from spatialyze.video_processor.stages.depth_estimation import DepthEstimation\n", + "\n", + "from spatialyze.video_processor.stages.detection_estimation import DetectionEstimation\n", + "from spatialyze.video_processor.stages.detection_estimation.segment_mapping import RoadPolygonInfo" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "3c4bddca-a1ae-4806-be85-8cf397d01ec9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from spatialyze.video_processor.stages.tracking.strongsort import StrongSORT as StageStrongSORT\n", + "from spatialyze.video_processor.stages.tracking_2d.strongsort import StrongSORT as StrongSORT2D" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "6737b34b-d928-45aa-940e-b23a7a6e5eb0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_road import FromTracking2DAndRoad\n", + "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_depth import FromTracking2DAndDepth\n", + "from spatialyze.video_processor.stages.tracking_3d.tracking_3d import Tracking3DResult, Tracking3D\n", + "from spatialyze.video_processor.stages.tracking_3d.from_tracking_2d_and_detection_3d import FromTracking2DAndDetection3D as FromT2DAndD3D\n", + "\n", + "# from spatialyze.video_processor.stages.segment_trajectory import SegmentTrajectory\n", + "# from spatialyze.video_processor.stages.segment_trajectory.construct_segment_trajectory import SegmentPoint\n", + "# from spatialyze.video_processor.stages.segment_trajectory.from_tracking_3d import FromTracking3D" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e9087777-3c98-414a-8f00-eda64e8f8ffd", + "metadata": {}, + "outputs": [], + "source": [ + "# Stream Processing\n", + "from spatialyze.video_processor.stream.decode_frame import DecodeFrame\n", + "from spatialyze.video_processor.stream.prune_frames import PruneFrames\n", + "from spatialyze.video_processor.stream.road_visibility_pruner import RoadVisibilityPruner\n", + "from spatialyze.video_processor.stream.mono_depth_estimator import MonoDepthEstimator\n", + "from spatialyze.video_processor.stream.yolo import Yolo\n", + "from spatialyze.video_processor.stream.object_type_pruner import ObjectTypePruner\n", + "from spatialyze.video_processor.stream.from_detection_2d_and_road import FromDetection2DAndRoad\n", + "from spatialyze.video_processor.stream.from_detection_2d_and_depth import FromDetection2DAndDepth\n", + "from spatialyze.video_processor.stream.exit_frame_sampler import ExitFrameSampler\n", + "from spatialyze.video_processor.stream.strongsort import StrongSORT" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "59907886", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# from spatialyze.video_processor.cache import disable_cache\n", + "# disable_cache()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e650e6ca-9d7c-41e9-98ee-f682b399040f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# from spatialyze.video_processor.utils.process_pipeline import format_trajectory, insert_trajectory, get_tracks\n", + "from spatialyze.video_processor.utils.prepare_trajectory import prepare_trajectory\n", + "# from spatialyze.video_processor.actions.tracking2d_overlay import tracking2d_overlay" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "264c8190-89ee-472a-bd73-d553eb3e3278", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from spatialyze.utils.ingest_road import ingest_road\n", + "from spatialyze.database import database, Database\n", + "# from spatialyze.legacy.world import empty_world\n", + "from spatialyze.utils import F\n", + "from spatialyze.predicate import camera, objects, lit, FindAllTablesVisitor, normalize, MapTablesTransformer, GenSqlVisitor\n", + "from spatialyze.data_types.camera import Camera as ACamera\n", + "from spatialyze.data_types.camera_config import CameraConfig as ACameraConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "355b8977", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "/data/apperception-data/processed/nuscenes/full-dataset-v1.0/Trainval\n" + ] + } + ], + "source": [ + "NUSCENES_PROCESSED_DATA = \"NUSCENES_PROCESSED_DATA\"\n", + "print(NUSCENES_PROCESSED_DATA in os.environ)\n", + "print(os.environ['NUSCENES_PROCESSED_DATA'])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "00c1dd1e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "DATA_DIR = os.environ[NUSCENES_PROCESSED_DATA]\n", + "# with open(os.path.join(DATA_DIR, \"videos\", \"frames.pkl\"), \"rb\") as f:\n", + "# videos = pickle.load(f)\n", + "with open(os.path.join(DATA_DIR, 'videos', 'videos.json'), 'r') as f:\n", + " videos = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "1f6981ac-60b4-43f4-9c3b-32a4e84e4aa1", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0523 0778 467\n" + ] + } + ], + "source": [ + "with open('./data/evaluation/video-samples/boston-seaport.txt', 'r') as f:\n", + " sampled_scenes = f.read().split('\\n')\n", + "print(sampled_scenes[0], sampled_scenes[-1], len(sampled_scenes))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "fcb3599c-3807-4044-9636-45b2d94fe7e8", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "BENCHMARK_DIR = \"./outputs/run\"\n", + "\n", + "\n", + "def bm_dir(*args: \"str\"):\n", + " return os.path.join(BENCHMARK_DIR, *args)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ad97a25a-0356-4d7e-9096-f26c00d2d9d4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def get_sql(predicate: \"PredicateNode\"):\n", + " tables, camera = FindAllTablesVisitor()(predicate)\n", + " tables = sorted(tables)\n", + " mapping = {t: i for i, t in enumerate(tables)}\n", + " predicate = normalize(predicate)\n", + " predicate = MapTablesTransformer(mapping)(predicate)\n", + "\n", + " t_tables = ''\n", + " t_outputs = ''\n", + " for i in range(len(tables)):\n", + " t_tables += '\\n' \\\n", + " 'JOIN Item_General_Trajectory ' \\\n", + " f'AS t{i} ' \\\n", + " f'ON Cameras.timestamp <@ t{i}.trajCentroids::period'\n", + " t_outputs += f', t{i}.itemId'\n", + "\n", + " return f\"\"\"\n", + " SELECT Cameras.frameNum {t_outputs}\n", + " FROM Cameras{t_tables}\n", + " WHERE\n", + " {GenSqlVisitor()(predicate)}\n", + " \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5d4149a3-43b5-4531-90dd-31dd795bdaa1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "slices = {\n", + " \"noopt\": (0, 1),\n", + " \"inview\": (1, 2),\n", + " \"objectfilter\": (2, 3),\n", + " \"geo\": (3, 4),\n", + " \"de\": (4, 5),\n", + " \"opt\": (5, 6),\n", + " # \"optde\": (6, 7),\n", + " 'dev': (0, 2),\n", + " 'freddie': (1, 2),\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "275836d5", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def run_benchmark(pipeline, filename, predicates, run=0, ignore_error=False):\n", + " print(filename)\n", + " metadata_strongsort = {}\n", + " metadata_d2d = {}\n", + " failed_videos = []\n", + "\n", + " all_metadata = {\n", + " 'detection': metadata_d2d,\n", + " 'sort': metadata_strongsort,\n", + " }\n", + " print('# of total videos:', len(videos))\n", + "\n", + " names = set(sampled_scenes[:1])\n", + " # names = set(sampled_scenes)\n", + " filtered_videos = [\n", + " n for n in videos\n", + " if n[6:10] in names and 'FRONT' in n # and n.endswith('FRONT')\n", + " ]\n", + " N = len(filtered_videos)\n", + " print('# of filtered videos:', N)\n", + "\n", + " # # s_from, s_to = slices[test]\n", + " # s_from, s_to = (int(test), int(test) + 1)\n", + " # STEP = math.ceil(N / 10)\n", + " # print('test', test)\n", + " # print('from', s_from*STEP)\n", + " # print('to ', s_to*STEP)\n", + " # filtered_videos = filtered_videos[s_from*STEP:min(s_to*STEP, N)]\n", + " print('# of sliced videos:', len(filtered_videos))\n", + " # ingest_road(database, './data/scenic/road-network/boston-seaport')\n", + "\n", + " for pre in [*all_metadata.keys(), 'qresult', 'performance', 'failedvideos']:\n", + " p = os.path.join(BENCHMARK_DIR, f\"{pre}--{filename}_{run}\")\n", + " if os.path.exists(p):\n", + " shutil.rmtree(p)\n", + " os.makedirs(p)\n", + "\n", + " def save_perf():\n", + " for n, message in failed_videos:\n", + " p = bm_dir(f'failedvideos--{filename}_{run}', f'{n}.txt')\n", + " with open(p, \"w\") as f:\n", + " f.write(message)\n", + "\n", + " for i, name in tqdm(enumerate(filtered_videos), total=len(filtered_videos)):\n", + " # try:\n", + " start_input = time.time()\n", + " with open(os.path.join(DATA_DIR, 'videos', 'boston-seaport-' + name + '.pkl'), 'rb') as f:\n", + " video = pickle.load(f)\n", + " video_filename = video['filename']\n", + "\n", + " frames = Video(\n", + " os.path.join(DATA_DIR, \"videos\", video[\"filename\"]),\n", + " [camera_config(*f, 0) for f in video[\"frames\"]],\n", + " )\n", + " time_input = time.time() - start_input\n", + "\n", + " t1 = time.time()\n", + " # inview = RoadVisibilityPruner(predicate=predicates[0], distance=50)\n", + " decode = DecodeFrame()\n", + " yolo = Yolo(decode)\n", + " dets = ObjectTypePruner(yolo, types=['car'])\n", + " # depths = MonoDepthEstimator(decode)\n", + " # d3ds = FromDetection2DAndDepth(dets, depths)\n", + " d3ds = FromDetection2DAndRoad(dets)\n", + " efp = ExitFrameSampler(d3ds)\n", + " # d3ds = PruneFrames(efp, d3ds)\n", + " # tracks = StrongSORT(d3ds, decode)\n", + " output1 = efp.execute(frames)\n", + "\n", + " t2 = time.time()\n", + "\n", + " pipeline = Pipeline()\n", + " # pipeline.add_filter(InView(predicate=predicates[0], distance=50))\n", + " pipeline.add_filter(StageDecodeFrame())\n", + " pipeline.add_filter(YoloDetection())\n", + " pipeline.add_filter(ObjectTypeFilter(types=['car']))\n", + " # pipeline.add_filter(DepthEstimation())\n", + " # pipeline.add_filter(StageFromDetection2DAndDepth())\n", + " pipeline.add_filter(StageFromDetection2DAndRoad())\n", + " pipeline.add_filter(DetectionEstimation())\n", + " # pipeline.add_filter(StageStrongSORT())\n", + " output2 = pipeline.run(Payload(frames))\n", + "\n", + " t3 = time.time()\n", + "\n", + " print('stream', t2 - t1)\n", + " print('stage ', t3 - t2)\n", + " # for o1, o2 in zip(output1, output2[StageFromDetection2DAndRoad]):\n", + " # assert np.array_equal(o1[0].cpu().numpy(), o2[0].cpu().numpy()), (o1[0], o2[0])\n", + " for idx, (o1, o2) in enumerate(zip(output1, output2.keep)):\n", + " # print(type(o1), type(o2))\n", + " assert (o1 is True) == (o2 == 1), (o1, o2, idx)\n", + "\n", + " # for o1, o2 in zip(decode.execute(frames), output2[StageDecodeFrame]):\n", + " # assert np.array_equal(o1, o2)\n", + "# d1 = {}\n", + "# d2 = {}\n", + "# for o in output1:\n", + "# tid = o[0].object_id\n", + "# assert tid not in d1\n", + "# d1[tid] = o\n", + "\n", + "# for o in output2[StageStrongSORT]:\n", + "# for det in o:\n", + "# tid = det.object_id\n", + "# if tid not in d2:\n", + "# d2[tid] = []\n", + "# d2[tid].append(det)\n", + "\n", + "# for tid, track1 in d1.items():\n", + "# track2 = d2[tid]\n", + "# assert len(track1) == len(track2), (len(track1), len(track2))\n", + "# for t1, t2 in zip(track1, track2):\n", + "# assert t1.detection_id == t2.detection_id, (t1.detection_id, t2.detection_id)\n", + " # for o1, o2 in zip(d3ds.execute(frames), output2[StageFromDetection2DAndRoad]):\n", + " # assert o1[0].shape == o2[0].shape, (o1[0].shape, o2[0].shape)\n", + " # assert np.array_equal(o1[0].cpu().numpy(), o2[0].cpu().numpy()), (o1[0], o2[0])\n", + " # output = pipeline.run(Payload(frames))\n", + "\n", + " # metadata_strongsort[name] = output[StrongSORT2D]\n", + " # metadata_d2d[name] = output[Detection2D]\n", + "\n", + " # for pre, metadata in all_metadata.items():\n", + " # p = bm_dir(f\"{pre}--{filename}_{run}\", f\"{name}.json\")\n", + " # with open(p, \"w\") as f:\n", + " # json.dump(metadata[name], f, cls=MetadataJSONEncoder,\n", + " # indent=1)\n", + " break\n", + "\n", + "# times_rquery = []\n", + "# for i, predicate in enumerate(predicates):\n", + "# start_rquery = time.time()\n", + "# database.reset(True)\n", + "\n", + "# # Ingest Trackings\n", + "# # ego_meta = frames.interpolated_frames\n", + "# # sortmeta = Tracking3D.get(output)\n", + "# # assert sortmeta is not None\n", + "# # segment_trajectories = FromTracking3D.get(output)\n", + "# # tracks = get_tracks(sortmeta, ego_meta, segment_trajectories)\n", + "# for track in output:\n", + "# obj_id = track[0].object_id\n", + "# trajectory = prepare_trajectory(name, obj_id, track, frames.camera_configs)\n", + "# if trajectory:\n", + "# insert_trajectory(database, *trajectory)\n", + "\n", + "# # Ingest Camera\n", + "# accs: 'ACameraConfig' = []\n", + "# for idx, cc in enumerate(frames.interpolated_frames):\n", + "# acc = ACameraConfig(\n", + "# frame_id=cc.frame_id,\n", + "# frame_num=idx,\n", + "# filename=cc.filename,\n", + "# camera_translation=cc.camera_translation,\n", + "# camera_rotation=cc.camera_rotation,\n", + "# camera_intrinsic=cc.camera_intrinsic,\n", + "# ego_translation=cc.ego_translation,\n", + "# ego_rotation=cc.ego_rotation,\n", + "# timestamp=cc.timestamp,\n", + "# cameraHeading=cc.camera_heading,\n", + "# egoHeading=cc.ego_heading,\n", + "# )\n", + "# accs.append(acc)\n", + "# camera = ACamera(accs, cc.camera_id)\n", + "# database.insert_cam(camera)\n", + "\n", + "# query = get_sql(predicate)\n", + "# qresult = database.execute(query)\n", + "\n", + "# p = bm_dir(f\"qresult--{filename}_{run}\", f\"{name}-{i}.json\")\n", + "# with open(p, 'w') as f:\n", + "# json.dump(qresult, f, indent=1)\n", + "# time_rquery = time.time() - start_rquery\n", + "# times_rquery.append(time_rquery)\n", + "# # runtime_query.append({'name': name, 'predicate': i, 'runtime': time_rquery})\n", + "\n", + "# # save video\n", + "# start_video = time.time()\n", + "# tracking2d_overlay(output, './tmp.mp4')\n", + "# time_video = time.time() - start_video\n", + "# # runtime_video.append({'name': name, 'runtime': time_video})\n", + "\n", + "# perf = []\n", + "# for stage in pipeline.stages:\n", + "# benchmarks = [*filter(\n", + "# lambda x: video['filename'] in x['name'],\n", + "# stage.benchmark\n", + "# )]\n", + "# assert len(benchmarks) == 1\n", + "# perf.append({\n", + "# 'stage': stage.classname(),\n", + "# 'benchmark': benchmarks[0]\n", + "# })\n", + "\n", + "# for bm in getattr(stage, '_benchmark', []):\n", + "# if video['filename'] in bm['name']:\n", + "# perf.append({\n", + "# 'stage': stage.classname(),\n", + "# 'addition': True,\n", + "# 'benchmark': bm,\n", + "# })\n", + "\n", + "# perf.append({\n", + "# 'stage': 'ingest',\n", + "# 'benchmark': {\n", + "# 'name': name,\n", + "# 'runtime': time_input\n", + "# }\n", + "# })\n", + "# perf.append({\n", + "# 'stage': 'save',\n", + "# 'benchmark': {\n", + "# 'name': name,\n", + "# 'runtime': time_video\n", + "# }\n", + "# })\n", + "# for i, time_rquery in enumerate(times_rquery):\n", + "# perf.append({\n", + "# 'stage': 'query',\n", + "# 'benchmark': {\n", + "# 'name': name,\n", + "# 'predicate': i,\n", + "# 'runtime': time_rquery\n", + "# }\n", + "# })\n", + "# p = bm_dir(f'performance--{filename}_{run}', f'{name}.json')\n", + "# with open(p, \"w\") as f:\n", + "# json.dump(perf, f, indent=1)\n", + "# except Exception as e:\n", + "# if ignore_error:\n", + "# message = str(traceback.format_exc())\n", + "# failed_videos.append((name, message))\n", + "# print(video_filename)\n", + "# print(e)\n", + "# print(message)\n", + "# print(\"------------------------------------------------------------------------------------\")\n", + "# print()\n", + "# print()\n", + "# else:\n", + "# raise e\n", + "\n", + "# if len(metadata_d2d) % 10 == 0:\n", + "# save_perf()\n", + "# save_perf()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "98283938-b68f-4925-a5ef-eee7c6c46c65", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " groundtruth_detection=False,\n", + " geo_depth=True,\n", + " detection_estimation=True,\n", + " strongsort=False,\n", + " ss_update_when_skip=True,\n", + " ss_cache=True,\n", + "):\n", + " pipeline = Pipeline()\n", + "\n", + " # In-View Filter\n", + " if in_view:\n", + " # TODO: view angle and road type should depends on the predicate\n", + " pipeline.add_filter(InView(50, predicate=predicate))\n", + "\n", + " # Decode\n", + " pipeline.add_filter(ParallelDecodeFrame())\n", + "\n", + " # 2D Detection\n", + " if groundtruth_detection:\n", + " with open(os.path.join(DATA_DIR, 'annotation_partitioned.pkl'), 'rb') as f:\n", + " df_annotations = pickle.load(f)\n", + " pipeline.add_filter(GroundTruthDetection(df_annotations))\n", + " else:\n", + " pipeline.add_filter(YoloDetection())\n", + "\n", + " # Object Filter\n", + " if object_filter:\n", + " pipeline.add_filter(ObjectTypeFilter(predicate=predicate))\n", + "\n", + " # 3D Detection\n", + " if geo_depth:\n", + " pipeline.add_filter(StageFromDetection2DAndRoad())\n", + " else:\n", + " pipeline.add_filter(DepthEstimation())\n", + " pipeline.add_filter(StageFromDetection2DAndDepth())\n", + "\n", + " # Detection Estimation\n", + " if detection_estimation:\n", + " pipeline.add_filter(DetectionEstimation())\n", + "\n", + " # Tracking\n", + " pipeline.add_filter(StrongSORT2D(\n", + " # method='update-empty' if ss_update_when_skip else 'increment-ages',\n", + " method='update-empty',\n", + " cache=ss_cache,\n", + " ))\n", + "\n", + " pipeline.add_filter(FromT2DAndD3D())\n", + " # if geo_depth:\n", + " # pipeline.add_filter(FromTracking2DAndRoad())\n", + " # else:\n", + " # pipeline.add_filter(FromTracking2DAndDepth())\n", + "\n", + " # Segment Trajectory\n", + " # pipeline.add_filter(FromTracking3D())\n", + "\n", + " return pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "f4adca3d-7963-4dc6-bde1-d0ce107ae959", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "p_noSSOpt = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=False,\n", + " geo_depth=False,\n", + " detection_estimation=False,\n", + " ss_cache=False,\n", + ")\n", + "\n", + "p_noOpt = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=False,\n", + " geo_depth=False,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_inview = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=False,\n", + " geo_depth=False,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_objectFilter = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=True,\n", + " geo_depth=False,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_geo = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=False,\n", + " geo_depth=True,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_de = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=False,\n", + " geo_depth=False,\n", + " detection_estimation=True\n", + ")\n", + "\n", + "p_deIncr = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=False,\n", + " object_filter=False,\n", + " geo_depth=False,\n", + " detection_estimation=True,\n", + " ss_update_when_skip=False,\n", + ")\n", + "\n", + "p_opt = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " geo_depth=True,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_optDe = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " geo_depth=True,\n", + " detection_estimation=True\n", + ")\n", + "\n", + "p_optIncr = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " geo_depth=True,\n", + " detection_estimation=False,\n", + " ss_update_when_skip=False,\n", + ")\n", + "\n", + "p_optDeIncr = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " geo_depth=True,\n", + " detection_estimation=True,\n", + " ss_update_when_skip=False,\n", + ")\n", + "\n", + "p_gtOpt = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " groundtruth_detection=True,\n", + " geo_depth=True,\n", + " detection_estimation=False\n", + ")\n", + "\n", + "p_gtOptDe = lambda predicate: create_pipeline(\n", + " predicate,\n", + " in_view=True,\n", + " object_filter=True,\n", + " groundtruth_detection=True,\n", + " geo_depth=True,\n", + " detection_estimation=True\n", + ")\n", + "\n", + "pipelines = {\n", + " \"nossopt\": p_noSSOpt,\n", + " \"noopt\": p_noOpt,\n", + " \"inview\": p_inview,\n", + " \"objectfilter\": p_objectFilter,\n", + " \"geo\": p_geo,\n", + " \"de\": p_de,\n", + " # \"deincr\": p_deIncr,\n", + " \"opt\": p_opt,\n", + " # \"optincr\": p_optIncr,\n", + " \"optde\": p_optDe,\n", + " # \"optdeincr\": p_optDeIncr,\n", + "\n", + " # \"gtopt\": p_gtOpt,\n", + " # \"gtoptde\": p_gtOptDe\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "653e586a-a98c-4c15-ac5a-17551b3155db", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# if test == 'dev':\n", + "# test = 'opt'" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "48f7c558-dc6e-4b86-b447-a3ffac74c966", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def run(__test):\n", + " o = objects[0]\n", + " c = camera\n", + " pred1 = (\n", + " (o.type == 'person') &\n", + " # F.contained(c.ego, 'intersection') &\n", + " F.contained(o.trans@c.time, 'intersection') &\n", + " F.angle_excluding(F.facing_relative(o.traj@c.time, c.cam), lit(-70), lit(70)) &\n", + " # F.angle_between(F.facing_relative(c.cam, F.road_direction(c.ego)), lit(-15), lit(15)) &\n", + " (F.distance(c.cam, o.traj@c.time) < lit(50)) # &\n", + " # (F.view_angle(o.trans@c.time, c.camAbs) < lit(35))\n", + " )\n", + " pred1_notrack = (\n", + " F.contained(o.trans@c.time, 'intersection') &\n", + " (F.distance(c.cam, o.traj@c.time) < lit(50)) &\n", + " (o.type == 'person')\n", + " )\n", + "\n", + " obj1 = objects[0]\n", + " obj2 = objects[1]\n", + " cam = camera\n", + " pred2 = (\n", + " (obj1.id != obj2.id) &\n", + " ((obj1.type == 'car') | (obj1.type == 'truck')) &\n", + " ((obj2.type == 'car') | (obj2.type == 'truck')) &\n", + " F.angle_between(F.facing_relative(cam.cam, F.road_direction(cam.cam)), -15, 15) &\n", + " (F.distance(cam.cam, obj1.trans@cam.time) < 50) &\n", + "\n", + " # (F.view_angle(obj1.trans@cam.time, cam.ego) < 70 / 2.0) &\n", + " (F.distance(cam.cam, obj2.trans@cam.time) < 50) &\n", + " # (F.view_angle(obj2.trans@cam.time, cam.ego) < 70 / 2.0) &\n", + " F.contains_all('intersection', [obj1.trans, obj2.trans]@cam.time) &\n", + " # F.angle_between(F.facing_relative(obj1.trans@cam.time, cam.cam), 40, 135) &\n", + " # F.angle_between(F.facing_relative(obj2.trans@cam.time, cam.cam), -135, -50)\n", + " F.angle_between(F.facing_relative(obj1.trans@cam.time, obj2.trans@cam.time), -180, -90)\n", + " # (F.min_distance(cam.ego, 'intersection') < 10) &\n", + " # F.angle_between(F.facing_relative(obj1.trans@cam.time, obj2.trans@cam.time), 100, -100)\n", + " )\n", + " pred2_notrack = (\n", + " (F.distance(cam.cam, obj1.trans@cam.time) < 50) &\n", + " (F.distance(cam.cam, obj2.trans@cam.time) < 50) &\n", + " F.contains_all('intersection', [obj1.trans, obj2.trans]@cam.time) &\n", + " (obj1.id != obj2.id) &\n", + " ((obj1.type == 'car') | (obj1.type == 'truck')) &\n", + " ((obj2.type == 'car') | (obj2.type == 'truck'))\n", + " )\n", + "\n", + " obj1 = objects[0]\n", + " cam = camera\n", + " pred3 = (\n", + " ((obj1.type == 'car') | (obj1.type == 'truck')) &\n", + " # (F.view_angle(obj1.trans@cam.time, cam.ego) < 70 / 2) &\n", + " F.angle_between(F.facing_relative(cam.cam, F.road_direction(cam.ego, cam.ego)), 135, 225) &\n", + " F.contained(cam.cam, F.road_segment('lane')) &\n", + " F.contained(obj1.trans@cam.time, F.road_segment('lane')) &\n", + " F.angle_between(F.facing_relative(obj1.trans@cam.time, F.road_direction(obj1.traj@cam.time, cam.ego)), -15, 15) &\n", + " # F.angle_between(F.facing_relative(obj1.trans@cam.time, cam.ego), 135, 225) &\n", + " (F.distance(cam.cam, obj1.trans@cam.time) < 10)\n", + " )\n", + " pred3_notrack = (\n", + " # F.contained(cam.cam, F.road_segment('lane')) &\n", + " F.contained(obj1.trans@cam.time, F.road_segment('lane')) &\n", + " (F.distance(cam.cam, obj1.trans@cam.time) < 10) &\n", + " ((obj1.type == 'car') | (obj1.type == 'truck'))\n", + " )\n", + "\n", + " cam = camera\n", + " car1 = objects[0]\n", + " opposite_car_1 = objects[1]\n", + " opposite_car_2 = objects[2]\n", + "\n", + " pred4 = (\n", + " ((car1.type == 'car') | (car1.type == 'truck')) &\n", + " ((opposite_car_2.type == 'car') | (opposite_car_2.type == 'truck')) &\n", + " ((opposite_car_1.type == 'car') | (opposite_car_1.type == 'truck')) &\n", + " (opposite_car_1.id != opposite_car_2.id) &\n", + " (car1.id != opposite_car_2.id) &\n", + " (car1.id != opposite_car_1.id) &\n", + "\n", + " F.contained(cam.cam, F.road_segment('lane')) &\n", + " F.contained(car1.trans@cam.time, F.road_segment('lane')) &\n", + " F.contained(opposite_car_1.trans@cam.time, F.road_segment('lane')) &\n", + " F.contained(opposite_car_2.trans@cam.time, F.road_segment('lane')) &\n", + " F.angle_between(F.facing_relative(cam.cam, F.road_direction(cam.cam, cam.cam)), -15, 15) &\n", + " # (F.view_angle(car1.traj@cam.time, cam.ego) < 70 / 2) &\n", + " (F.distance(cam.cam, car1.traj@cam.time) < 40) &\n", + " F.angle_between(F.facing_relative(car1.traj@cam.time, cam.ego), -15, 15) &\n", + " # F.angle_between(F.facing_relative(car1.traj@cam.time, F.road_direction(car1.traj@cam.time, cam.ego)), -15, 15) &\n", + " F.ahead(car1.traj@cam.time, cam.cam) &\n", + " # (F.convert_camera(opposite_car.traj@cam.time, cam.ego) > [-10, 0]) &\n", + " # (F.convert_camera(opposite_car.traj@cam.time, cam.ego) < [-1, 50]) &\n", + " F.angle_between(F.facing_relative(opposite_car_1.traj@cam.time, cam.ego), 135, 225) &\n", + " # (F.distance(opposite_car@cam.time, car2@cam.time) < 40)# &\n", + " F.angle_between(F.facing_relative(opposite_car_2.traj@cam.time, opposite_car_1.traj@cam.time), -15, 15) &\n", + " F.angle_between(F.facing_relative(opposite_car_2.traj@cam.time, F.road_direction(opposite_car_2.traj@cam.time, cam.ego)), -15, 15) &\n", + " F.ahead(opposite_car_2.traj@cam.time, opposite_car_1.traj@cam.time)\n", + " )\n", + " pred4_notrack = (\n", + " # F.contained(cam.cam, F.road_segment('lane')) &\n", + " F.contained(car1.trans@cam.time, F.road_segment('lane')) &\n", + " F.contained(opposite_car_1.trans@cam.time, F.road_segment('lane')) &\n", + " F.contained(opposite_car_2.trans@cam.time, F.road_segment('lane')) &\n", + " ((car1.type == 'car') | (car1.type == 'truck')) &\n", + " ((opposite_car_2.type == 'car') | (opposite_car_2.type == 'truck')) &\n", + " ((opposite_car_1.type == 'car') | (opposite_car_1.type == 'truck')) &\n", + " (opposite_car_1.id != opposite_car_2.id) &\n", + " (car1.id != opposite_car_2.id) &\n", + " (car1.id != opposite_car_1.id)\n", + " )\n", + "\n", + " p1 = pipelines[__test](pred1)\n", + " p2 = pipelines[__test](pred2)\n", + " p34 = pipelines[__test](pred3)\n", + "\n", + " print('Pipeline P2:')\n", + " for s in p2.stages:\n", + " print(' -', s)\n", + " run_benchmark(p2, 'q2-' + __test, [pred2, pred2_notrack], run=1, ignore_error=True)\n", + "\n", + " print('Pipeline P3,P4:')\n", + " for s in p34.stages:\n", + " print(' -', s)\n", + " run_benchmark(p34, 'q34-' + __test, [pred3, pred4, pred3_notrack, pred4_notrack], run=1, ignore_error=True)\n", + "\n", + " if __test != 'optde' and __test != 'de':\n", + " print('Pipeline P1:')\n", + " for s in p1.stages:\n", + " print(' -', s)\n", + " run_benchmark(p1, 'q1-' + __test, [pred1, pred1_notrack], run=1, ignore_error=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "811c4351-f2a7-478b-b264-69a3e8d75c69", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------- 0 / 4 --- de -----------\n", + "Pipeline P2:\n", + " - DecodeFrame.ParallelDecodeFrame\n", + " - Detection2D.YoloDetection\n", + " - DepthEstimation\n", + " - Detection3D.FromDetection2DAndDepth\n", + " - DetectionEstimation\n", + " - Tracking2D.StrongSORT\n", + " - Tracking3D.FromTracking2DAndDetection3D\n", + "q2-de\n", + "# of total videos: 5100\n", + "# of filtered videos: 3\n", + "# of sliced videos: 3\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "493d1248b25d4a318cefa8b6ceb94821", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/3 [00:00 bool: + try: + shell = get_ipython().__class__.__name__ + if shell == 'ZMQInteractiveShell': + # Jupyter notebook or qtconsole + return True + elif shell == 'TerminalInteractiveShell': + # Terminal running IPython + return False + else: + # Other type (?) + return False + except NameError: + # Probably standard Python interpreter + return False + + +if is_notebook(): + get_ipython().run_line_magic('cd', '../..') + from tqdm.notebook import tqdm + from nbutils.report_progress import report_progress +else: + from tqdm import tqdm + from evaluation.ablation.nbutils.report_progress import report_progress + + +# In[4]: + + +NUSCENES_PROCESSED_DATA = "NUSCENES_PROCESSED_DATA" +print(NUSCENES_PROCESSED_DATA in os.environ) +print(os.environ['NUSCENES_PROCESSED_DATA']) + + +# In[12]: + + +DATA_DIR = os.environ[NUSCENES_PROCESSED_DATA] +# with open(os.path.join(DATA_DIR, "videos", "frames.pkl"), "rb") as f: +# videos = pickle.load(f) +with open(os.path.join(DATA_DIR, 'videos', 'videos.json'), 'r') as f: + videos = json.load(f) +type(videos) +videos[0] + + +# In[10]: + + +with open('./data/evaluation/video-samples/boston-seaport.txt', 'r') as f: + sampled_scenes = f.read().split('\n') +print(sampled_scenes[0], sampled_scenes[-1], len(sampled_scenes)) + + +# In[17]: + + +with open(os.path.join(DATA_DIR, 'videos', 'boston-seaport-scene-' + sampled_scenes[0] + '-CAM_FRONT.pkl'), 'rb') as f: + video = pickle.load(f) +video_filename = video['filename'] + + +# In[18]: + + +video_filename + + +# In[25]: + + +frames = [] +cap = cv2.VideoCapture(os.path.join(DATA_DIR, 'videos', video_filename)) +while cap.isOpened(): + ret, frame = cap.read() + if not ret: + break + frames.append(frame) +cap.release() +cv2.destroyAllWindows() + + +# In[28]: + + +result = cv2.VideoWriter('./loop-video.avi', + cv2.VideoWriter_fourcc(*'MJPG'), + 12, (1600, 900)) +for i in tqdm(range(633)): + for frame in frames: + result.write(frame) +result.release() +cv2.destroyAllWindows() + + +# In[ ]: + + + +