diff --git a/codecov.yml b/codecov.yml index 06e536e..9db08aa 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,4 @@ ignore: - "spatialyze/video_processor/modules" - "spatialyze/video_processor/stages/detection_estimation/*" - - "spatialyze/video_processor/stages/segment_trajectory/*" - - "spatialyze/video_processor/stages/strongsort_with_skip.py" - "**/__init__.py" diff --git a/data/pipeline/test-results/DetectionEstimationKeep--scene-0655-CAM_FRONT.json b/data/pipeline/test-results/DetectionEstimationKeep--scene-0655-CAM_FRONT.json new file mode 100644 index 0000000..d2bfb13 --- /dev/null +++ b/data/pipeline/test-results/DetectionEstimationKeep--scene-0655-CAM_FRONT.json @@ -0,0 +1,239 @@ +[ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 1, + 0 +] \ No newline at end of file diff --git a/data/pipeline/test-results/DetectionEstimationKeep--scene-0757-CAM_FRONT.json b/data/pipeline/test-results/DetectionEstimationKeep--scene-0757-CAM_FRONT.json new file mode 100644 index 0000000..66b7223 --- /dev/null +++ b/data/pipeline/test-results/DetectionEstimationKeep--scene-0757-CAM_FRONT.json @@ -0,0 +1,239 @@ +[ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 +] \ No newline at end of file diff --git a/spatialyze/video_processor/stages/strongsort_with_skip.py b/evaluation/ablation/strongsort_with_skip.py similarity index 93% rename from spatialyze/video_processor/stages/strongsort_with_skip.py rename to evaluation/ablation/strongsort_with_skip.py index ce93776..f6aa481 100644 --- a/spatialyze/video_processor/stages/strongsort_with_skip.py +++ b/evaluation/ablation/strongsort_with_skip.py @@ -5,13 +5,13 @@ import numpy.typing as npt import torch -from ..modules.yolo_tracker.trackers.multi_tracker_zoo import StrongSORT, create_tracker -from ..modules.yolo_tracker.yolov5.utils.torch_utils import select_device -from ..payload import Payload -from ..types import DetectionId -from .decode_frame.decode_frame import DecodeFrame -from .detection_2d.detection_2d import Detection2D, Metadatum -from .stage import Stage +from spatialyze.video_processor.modules.yolo_tracker.trackers.multi_tracker_zoo import StrongSORT, create_tracker +from spatialyze.video_processor.modules.yolo_tracker.yolov5.utils.torch_utils import select_device +from spatialyze.video_processor.payload import Payload +from spatialyze.video_processor.types import DetectionId +from spatialyze.video_processor.stages.decode_frame.decode_frame import DecodeFrame +from spatialyze.video_processor.stages.detection_2d.detection_2d import Detection2D, Metadatum +from spatialyze.video_processor.stages.stage import Stage FILE = Path(__file__).resolve() SPATIALYZE = FILE.parent.parent.parent.parent diff --git a/pyproject.toml b/pyproject.toml index 526a975..e7a2069 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,4 @@ exclude = 'spatialyze/video_processor/modules' pythonVersion = '3.10' ignore = [ 'spatialyze/video_processor/modules', - 'spatialyze/video_processor/stages/detection_2d/ground_truth.py', - 'spatialyze/video_processor/stages/detection_estimation', ] diff --git a/spatialyze/utils/export_tables.py b/scripts/export_tables.py similarity index 100% rename from spatialyze/utils/export_tables.py rename to scripts/export_tables.py diff --git a/scripts/generate_pg_extender.py b/scripts/generate_pg_extender.py index efa06a1..4efa7e0 100644 --- a/scripts/generate_pg_extender.py +++ b/scripts/generate_pg_extender.py @@ -2,7 +2,7 @@ with open('./install.sql', 'w') as f: f.write('\n'.join(map( - lambda x: f'\i {x};', + lambda x: f'\\i {x};', sorted(filter( lambda x: x.endswith('sql') and x != 'install.sql', os.listdir('.'), diff --git a/spatialyze/utils/import_pickle.py b/scripts/import_pickle.py similarity index 100% rename from spatialyze/utils/import_pickle.py rename to scripts/import_pickle.py diff --git a/scripts/import_tables.py b/scripts/import_tables.py index e1920d5..fdb49a2 100644 --- a/scripts/import_tables.py +++ b/scripts/import_tables.py @@ -11,7 +11,6 @@ Database, database, columns, - place_holder, ) @@ -88,6 +87,10 @@ def _name(column: "tuple[str, str]") -> str: return column[0] +def place_holder(num: int): + return ",".join(["%s"] * num) + + def _insert_into_camera(database: "Database", value: tuple, commit=True): cursor = database.connection.cursor() cursor.execute( diff --git a/scripts/ingest_data_csv.ipynb b/scripts/ingest_data_csv.ipynb index 816c466..b5f6e9b 100644 --- a/scripts/ingest_data_csv.ipynb +++ b/scripts/ingest_data_csv.ipynb @@ -15,8 +15,9 @@ "metadata": {}, "outputs": [], "source": [ - "from spatialyze.database import Database, database\n", - "from spatialyze.utils import import_tables, ingest_road" + "from spatialyze.database import database\n", + "from spatialyze.utils import ingest_road\n", + "from import_tables import import_tables" ] }, { diff --git a/scripts/ingest_data_from_file.py b/scripts/ingest_data_from_file.py index da27c26..13a719d 100644 --- a/scripts/ingest_data_from_file.py +++ b/scripts/ingest_data_from_file.py @@ -1,8 +1,8 @@ # %% -from spatialyze.database import Database, database -from spatialyze.utils import import_tables, ingest_road, import_pickle +from import_pickle import import_pickle +from spatialyze.database import database # %% CSV_DATA_PATH = 'data/scenic/database' diff --git a/spatialyze/database.py b/spatialyze/database.py index 4089e72..d1e9ace 100644 --- a/spatialyze/database.py +++ b/spatialyze/database.py @@ -97,10 +97,6 @@ def _schema(column: "tuple[str, str]") -> str: return " ".join(column) -def place_holder(num: int): - return ",".join(["%s"] * num) - - class Database: connection: "Connection" cursor: "Cursor" diff --git a/spatialyze/utils/F/same_region.py b/spatialyze/utils/F/same_region.py index f3a354e..7f1daef 100644 --- a/spatialyze/utils/F/same_region.py +++ b/spatialyze/utils/F/same_region.py @@ -1,7 +1,7 @@ -import ast - -from ...predicate import GenSqlVisitor, PredicateNode, call_node +from ...predicate import GenSqlVisitor, LiteralNode, PredicateNode, call_node from .common import ROAD_TYPES +from .common import default_location as dl +from .common import is_location_type @call_node @@ -12,7 +12,9 @@ def same_region( ): assert kwargs is None or len(kwargs) == 0, kwargs type_, traj1, traj2 = args - if not isinstance(type_, ast.Constant) or type_.value.lower() not in ROAD_TYPES: + if not isinstance(type_, LiteralNode) or type_.value.lower() not in ROAD_TYPES: raise Exception(f"Unsupported road type: {type_}") - return f"sameRegion({','.join(map(visitor, [type_, traj1, traj2]))})" + assert is_location_type(traj1), type(traj1) + assert is_location_type(traj2), type(traj2) + return f"sameRegion({','.join(map(visitor, [type_, dl(traj1), dl(traj2)]))})" diff --git a/spatialyze/utils/__init__.py b/spatialyze/utils/__init__.py index 27f6679..7b73f26 100644 --- a/spatialyze/utils/__init__.py +++ b/spatialyze/utils/__init__.py @@ -1,13 +1,9 @@ from . import F from .df_to_camera_config import df_to_camera_config -from .export_tables import export_tables -from .import_pickle import import_pickle from .ingest_road import ingest_road __all__ = [ "F", "df_to_camera_config", - "import_pickle", - "export_tables", "ingest_road", ] diff --git a/spatialyze/utils/ingest_processed_nuscenes.py b/spatialyze/utils/ingest_processed_nuscenes.py index 8e311f8..f6fd82d 100644 --- a/spatialyze/utils/ingest_processed_nuscenes.py +++ b/spatialyze/utils/ingest_processed_nuscenes.py @@ -32,7 +32,7 @@ def ingest_processed_nuscenes( camera_map: "dict[CameraKey, list[NuscenesCamera]]", database: "Database", ): - keys = list(camera_map.keys()) + keys = [k for k in camera_map.keys() if camera_map[k][0].location == "boston-seaport"] # print(len(keys)) # ks = [k for k in keys if camera_map[k][0].location == 'boston-seaport'] # print(len(ks)) @@ -44,8 +44,6 @@ def ingest_processed_nuscenes( print("Ingesting Cameras and Annotations") for k in tqdm(ks, total=len(ks)): camera = camera_map[k] - if camera[0].location != "boston-seaport": - continue annotations = annotations_map[k] objects: "dict[str, _MovableObject]" = {} diff --git a/spatialyze/utils/ingest_road.py b/spatialyze/utils/ingest_road.py index a79997f..09971bf 100644 --- a/spatialyze/utils/ingest_road.py +++ b/spatialyze/utils/ingest_road.py @@ -694,11 +694,3 @@ def ingest_road(database: "Database", directory: str): add_segment_type(database, ROAD_TYPES) database.reset() - - -if __name__ == "__main__": - import sys - - from spatialyze.database import database - - ingest_road(database, sys.argv[1]) diff --git a/spatialyze/video_processor/stages/detection_2d/ground_truth.py b/spatialyze/video_processor/stages/detection_2d/ground_truth.py index 8a55381..08c4e5c 100644 --- a/spatialyze/video_processor/stages/detection_2d/ground_truth.py +++ b/spatialyze/video_processor/stages/detection_2d/ground_truth.py @@ -1,3 +1,5 @@ +from typing import Any + import numpy as np import numpy.typing as npt import pandas as pd @@ -10,16 +12,18 @@ from .detection_2d import Detection2D, Metadatum signs = [-1, 1] -gp = [] +gp: list[tuple[int, int, int]] = [] for x in signs: for y in signs: for z in signs: - gp.append([x, y, z]) + gp.append((x, y, z)) -get_points = np.array(gp) +get_points = np.array(gp, dtype=np.int8) -def rotate(vectors: "npt.NDArray", rotation: "Quaternion") -> "npt.NDArray": +def rotate( + vectors: npt.NDArray[np.floating[Any]], rotation: Quaternion +) -> npt.NDArray[np.floating[Any]]: """Rotate 3D Vector by rotation quaternion. Params: vectors: (3 x N) 3-vectors each specified as any ordered @@ -33,19 +37,19 @@ def rotate(vectors: "npt.NDArray", rotation: "Quaternion") -> "npt.NDArray": def _3d_to_2d( - _translation: "Float3", - _size: "Float3", - _rotation: "Float4", - _camera_translation: "Float3", - _camera_rotation: "Quaternion", - _camera_intrinsics: "Float33", -) -> "Float4": - translation = np.array(_translation) - size = np.array((_size[1], _size[0], _size[2])) / 2.0 + _translation: Float3, + _size: Float3, + _rotation: Float4, + _camera_translation: Float3, + _camera_rotation: Quaternion, + _camera_intrinsics: Float33, +) -> Float4: + translation = np.array(_translation, dtype=np.float64) + size = np.array((_size[1], _size[0], _size[2]), dtype=np.float64) / 2.0 rotation = Quaternion(_rotation) - camera_translation = np.array(_camera_translation) + camera_translation = np.array(_camera_translation, dtype=np.float64) camera_rotation = _camera_rotation - camera_intrinsics = np.array(_camera_intrinsics) + camera_intrinsics = np.array(_camera_intrinsics, dtype=np.float64) points = size * get_points @@ -56,8 +60,8 @@ def _3d_to_2d( pixels = camera_intrinsics @ points_from_camera pixels /= pixels[2:3] - xs = pixels[0].tolist() - ys = pixels[1].tolist() + xs: list[float] = pixels[0].tolist() + ys: list[float] = pixels[1].tolist() left = min(xs) top = min(ys) diff --git a/spatialyze/video_processor/stages/detection_estimation/__init__.py b/spatialyze/video_processor/stages/detection_estimation/__init__.py index 3b7828a..905a0bf 100644 --- a/spatialyze/video_processor/stages/detection_estimation/__init__.py +++ b/spatialyze/video_processor/stages/detection_estimation/__init__.py @@ -13,8 +13,8 @@ from ...types import DetectionId, obj_detection from ...video import Video from ..detection_2d.detection_2d import Detection2D -from ..detection_2d.detection_2d import Metadatum as D2DMetadatum from ..detection_3d import Detection3D +from ..detection_3d import Metadatum as D3DMetadatum from ..in_view.in_view import get_views from ..stage import Stage from .detection_estimation import ( @@ -161,7 +161,7 @@ def _run(self, payload: "Payload"): return keep, {DetectionEstimation.classname(): metadata} -def new_car(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"): +def new_car(dets: "list[D3DMetadatum]", cur: "int", nxt: "int"): len_det = len(dets[cur][0]) for j in range(cur + 1, min(nxt + 1, len(dets))): future_det = dets[j][0] @@ -170,7 +170,7 @@ def new_car(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"): return min(nxt, len(dets) - 1) -def objects_count_change(dets: "list[D2DMetadatum]", cur: "int", nxt: "int"): +def objects_count_change(dets: "list[D3DMetadatum]", cur: "int", nxt: "int"): det, _, _ = dets[cur] for j in range(cur + 1, min(nxt + 1, len(dets))): future_det, _, _ = dets[j] @@ -243,7 +243,7 @@ def construct_estimated_all_detection_info( detection_ids: "list[DetectionId]", ego_config: "CameraConfig", ego_trajectory: "list[trajectory_3d]", -) -> "tuple[list[DetectionInfo], float]": +) -> "tuple[list[DetectionInfo], list[float]]": _times = time.time() all_detections = [] for det, did in zip(detections, detection_ids): @@ -260,7 +260,9 @@ def construct_estimated_all_detection_info( left3d, right3d = bbox3d[:3], bbox3d[3:] car_loc3d = tuple(map(float, (left3d + right3d) / 2)) assert len(car_loc3d) == 3 - car_bbox3d = (tuple(map(float, left3d)), tuple(map(float, right3d))) + lx, ly, lz = map(float, left3d) + rx, ry, rz = map(float, right3d) + car_bbox3d = ((lx, ly, lz), (rx, ry, rz)) all_detections.append(obj_detection(did, car_loc3d, car_loc2d, car_bbox3d, car_bbox2d)) all_detection_info, times = construct_all_detection_info( ego_config, ego_trajectory, all_detections diff --git a/spatialyze/video_processor/stages/detection_estimation/detection_estimation.py b/spatialyze/video_processor/stages/detection_estimation/detection_estimation.py index cc58121..c58449d 100644 --- a/spatialyze/video_processor/stages/detection_estimation/detection_estimation.py +++ b/spatialyze/video_processor/stages/detection_estimation/detection_estimation.py @@ -49,7 +49,7 @@ class DetectionInfo: car_bbox2d: "Float22" ego_trajectory: "list[trajectory_3d]" ego_config: "CameraConfig" - ego_road_polygon_info: "RoadPolygonInfo" + ego_road_polygon_info: "RoadPolygonInfo | None" timestamp: "datetime.datetime" = field(init=False) road_type: str = field(init=False) # distance: float = field(init=False) @@ -70,11 +70,11 @@ def __post_init__(self): # self.compute_geo_info() # self.compute_priority() - @property - def segment_line(self): - if self._to_compute_geo_info: - self.compute_geo_info() - return self._segment_line + # @property + # def segment_line(self): + # if self._to_compute_geo_info: + # self.compute_geo_info() + # return self._segment_line @property def segment_heading(self): @@ -167,9 +167,10 @@ def update_next_sample_frame_num(self): if car_exit_segment_action.invalid: return - car_exit_segment_frame_num = self.find_closest_frame_num( - car_exit_segment_action.finish_time - ) + finish_time = car_exit_segment_action.finish_time + assert finish_time is not None + car_exit_segment_frame_num = self.find_closest_frame_num(finish_time) + assert car_exit_segment_frame_num is not None next_sample_frame_num = min(next_sample_frame_num, car_exit_segment_frame_num) if next_sample_frame_num <= self.next_frame_num: return @@ -178,50 +179,48 @@ def update_next_sample_frame_num(self): car_exit_view_frame_num = get_car_exits_view_frame_num( detection_info, self.ego_views, next_sample_frame_num, self.fps ) + assert car_exit_view_frame_num is not None next_sample_frame_num = min(next_sample_frame_num, car_exit_view_frame_num) self.next_frame_num = next_sample_frame_num - def generate_sample_plan(self, view_distance: float = 50.0): - assert self.all_detection_info is not None - for detection_info in self.all_detection_info: - priority, sample_action = detection_info.generate_single_sample_action(view_distance) - if sample_action is not None: - self.add(priority, sample_action) - if self.action and not self.action.invalid: - self.next_frame_num = min( - self.next_frame_num, self.find_closest_frame_num(self.action.finish_time) - ) - - def add(self, priority: float, sample_action: "Action", time_threshold: float = 0.5): - assert sample_action is not None - if sample_action.invalid: - return - # assert not sample_action.invalid - - assert (self.action is None) == (self.current_priority is None) - if self.action is None or self.current_priority is None: - self.current_priority = priority - self.action = sample_action - else: - if sample_action.estimated_time < self.action.estimated_time: - if ( - priority >= self.current_priority - or sample_action.estimated_time / self.action.estimated_time < time_threshold - ): - self.current_priority = priority - self.action = sample_action - - def get_action(self): - return self.action + # def generate_sample_plan(self, view_distance: float = 50.0): + # assert self.all_detection_info is not None + # for detection_info in self.all_detection_info: + # priority, sample_action = detection_info.generate_single_sample_action(view_distance) + # if sample_action is not None: + # self.add(priority, sample_action) + # if self.action and not self.action.invalid: + # self.next_frame_num = min( + # self.next_frame_num, self.find_closest_frame_num(self.action.finish_time) + # ) + + # def add(self, priority: float, sample_action: "Action", time_threshold: float = 0.5): + # assert sample_action is not None + # if sample_action.invalid: + # return + # # assert not sample_action.invalid + + # assert (self.action is None) == (self.current_priority is None) + # if self.action is None or self.current_priority is None: + # self.current_priority = priority + # self.action = sample_action + # else: + # if sample_action.estimated_time < self.action.estimated_time: + # if ( + # priority >= self.current_priority + # or sample_action.estimated_time / self.action.estimated_time < time_threshold + # ): + # self.current_priority = priority + # self.action = sample_action + + # def get_action(self): + # return self.action def get_action_type(self): - if self.action is None: - return None - return self.action.action_type + return self.action and self.action.action_type def find_closest_frame_num(self, finish_time: "datetime.datetime"): - if finish_time is None: - return None + assert finish_time is not None nearest_index = None min_diff = None diff --git a/spatialyze/video_processor/stages/detection_estimation/sample_plan_algorithms.py b/spatialyze/video_processor/stages/detection_estimation/sample_plan_algorithms.py index f5ef796..e9e4ca2 100644 --- a/spatialyze/video_processor/stages/detection_estimation/sample_plan_algorithms.py +++ b/spatialyze/video_processor/stages/detection_estimation/sample_plan_algorithms.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from typing import Literal -from ...types import Float2, Float3 +from ...types import DetectionId, Float2, Float3, Float22 """ Action: @@ -18,18 +18,18 @@ CAR_EXIT_SEGMENT: "ActionType" = "car_exit_segment" EXIT_VIEW: "ActionType" = "exit_view" MEET_UP: "ActionType" = "meet_up" -OBJ_BASED_ACTION: "tuple[ActionType, ActionType]" = (CAR_EXIT_SEGMENT, EXIT_VIEW, MEET_UP) +OBJ_BASED_ACTION: "list[ActionType]" = [CAR_EXIT_SEGMENT, EXIT_VIEW, MEET_UP] @dataclass class Action: start_time: "datetime.datetime" - finish_time: "datetime.datetime" + finish_time: "datetime.datetime | None" start_loc: "Float2 | Float3" # TODO: should either be Float2 or Float3 - end_loc: "Float2 | Float3" # TODO: should either be Float2 or Float3 + end_loc: "Float2 | Float3 | None" # TODO: should either be Float2 or Float3 action_type: "ActionType" - target_obj_id: "str | None" = None - target_obj_bbox: "Float2 | None" = None + target_obj_id: "DetectionId | None" = None + target_obj_bbox: "Float22 | None" = None invalid: bool = field(init=False) estimated_time: "datetime.timedelta" = field(init=False) diff --git a/spatialyze/video_processor/stages/detection_estimation/segment_mapping.py b/spatialyze/video_processor/stages/detection_estimation/segment_mapping.py index 6be4b04..bb965a8 100644 --- a/spatialyze/video_processor/stages/detection_estimation/segment_mapping.py +++ b/spatialyze/video_processor/stages/detection_estimation/segment_mapping.py @@ -223,7 +223,9 @@ def get_detection_polygon_mapping(detections: "list[obj_detection]", ego_config: p = swkb.loads(roadpolygon.to_ewkb(), hex=True) assert isinstance(p, sg.Polygon) - XYs: "Tuple[array.array[float], array.array[float]]" = p.exterior.xy + exterior = p.exterior + assert hasattr(exterior, "xy") + XYs: "Tuple[array.array[float], array.array[float]]" = getattr(exterior, "xy") assert isinstance(XYs, tuple) assert isinstance(XYs[0], array.array), type(XYs[0]) assert isinstance(XYs[1], array.array), type(XYs[1]) diff --git a/spatialyze/video_processor/stages/detection_estimation/utils.py b/spatialyze/video_processor/stages/detection_estimation/utils.py index fe43f00..ebb3b07 100644 --- a/spatialyze/video_processor/stages/detection_estimation/utils.py +++ b/spatialyze/video_processor/stages/detection_estimation/utils.py @@ -9,8 +9,9 @@ import shapely.geometry import shapely.wkb +from ...types import Float2, Float3, Float22 + if TYPE_CHECKING: - from ...types import Float2, Float3, Float22 from .detection_estimation import DetectionInfo from .segment_mapping import RoadPolygonInfo @@ -51,14 +52,10 @@ def mph_to_mps(mph: "float"): MAX_CAR_SPEED.update({k: mph_to_mps(v) for k, v in MAX_CAR_SPEED.items()}) -def time_elapse(current_time, elapsed_time): +def time_elapse(current_time: datetime.datetime, elapsed_time: float): return current_time + datetime.timedelta(seconds=elapsed_time) -def compute_area(polygon) -> float: - return shapely.geometry.box(*polygon).area - - def compute_distance(loc1, loc2) -> float: return shapely.geometry.Point(loc1).distance(shapely.geometry.Point(loc2)) @@ -78,8 +75,9 @@ def car_move(car_loc, car_heading, car_speed, duration): def project_point_onto_linestring( - point: "shapely.geometry.Point", line: "shapely.geometry.LineString" -) -> "shapely.geometry.Point": + point: "shapely.geometry.Point", + line: "shapely.geometry.LineString", +) -> "shapely.geometry.Point | None": x: "npt.NDArray[np.float64]" = np.array(point.coords[0]) assert x.dtype == np.dtype(np.float64) @@ -100,7 +98,8 @@ def project_point_onto_linestring( def _construct_extended_line( - polygon: "shapely.geometry.Polygon | list[Float2] | list[Float3]", line: "Float22" + polygon: "shapely.geometry.Polygon | list[Float2] | list[Float3]", + line: "Float22", ): """ line: represented by 2 points @@ -108,32 +107,30 @@ def _construct_extended_line( """ try: polygon = shapely.geometry.Polygon(polygon) - minx, miny, maxx, maxy = polygon.bounds + minx, miny, maxx, maxy = list(polygon.bounds) except BaseException: assert isinstance(polygon, tuple) or isinstance(polygon, list) assert len(polygon) <= 2 if len(polygon) == 2: try: - a, b = shapely.geometry.LineString(polygon).boundary.geoms - minx = min(a.x, b.x) - maxx = max(a.x, b.x) - miny = min(a.y, b.y) - maxy = max(a.y, b.y) + boundary = shapely.geometry.LineString(polygon).boundary + assert hasattr(boundary, "geoms") + a, b = getattr(boundary, "geoms") + minx, maxx = sorted([a.x, b.x]) + miny, maxy = sorted([a.y, b.y]) except BaseException: assert polygon[0] == polygon[1] - minx = polygon[0][0] - maxx = polygon[0][0] - miny = polygon[0][1] - maxy = polygon[0][1] + minx, miny = polygon[0][0], polygon[0][1] + maxx, maxy = minx, miny else: - minx = polygon[0][0] - maxx = polygon[0][0] - miny = polygon[0][1] - maxy = polygon[0][1] + minx, miny = polygon[0][0], polygon[0][1] + maxx, maxy = minx, miny - line = shapely.geometry.LineString(line) + _line = shapely.geometry.LineString(line) bounding_box = shapely.geometry.box(minx, miny, maxx, maxy) - a, b = line.boundary.geoms + _boundary = _line.boundary + assert hasattr(_boundary, "geoms") + a, b = getattr(_boundary, "geoms") if a.x == b.x: # vertical line extended_line = shapely.geometry.LineString([(a.x, miny), (a.x, maxy)]) elif a.y == b.y: # horizonthal line @@ -159,7 +156,8 @@ def _construct_extended_line( def line_to_polygon_intersection( - polygon: "shapely.geometry.Polygon", line: "Float22" + polygon: "shapely.geometry.Polygon", + line: "Float22", ) -> "list[Float2]": """Find the intersection between a line and a polygon.""" try: @@ -239,7 +237,11 @@ def get_segment_line(road_segment_info: "RoadPolygonInfo", car_loc3d: "Float3"): def time_to_exit_current_segment( - detection_info: "DetectionInfo", current_time, car_loc, car_trajectory=None, is_ego=False + detection_info: "DetectionInfo", + current_time: datetime.datetime, + car_loc: Float3, + # car_trajectory=None, + # is_ego=False, ): """Return the time that the car exit the current segment @@ -247,37 +249,41 @@ def time_to_exit_current_segment( car heading is the same as road heading car drives at max speed if no trajectory is given """ - if is_ego: - current_polygon_info = detection_info.ego_road_polygon_info - polygon = current_polygon_info.polygon - else: - current_polygon_info = detection_info.road_polygon_info - polygon = current_polygon_info.polygon - - if car_trajectory: - for point in car_trajectory: - if point.timestamp > current_time and not shapely.geometry.Polygon(polygon).contains( - shapely.geometry.Point(point.coordinates[:2]) - ): - return point.timestamp, point.coordinates[:2] - return None, None - if detection_info.road_type == "intersection": - return None, None - if detection_info.segment_heading is None and detection_info.road_type != "intersection": + # if is_ego: + # current_polygon_info = detection_info.ego_road_polygon_info + # polygon = current_polygon_info.polygon + # else: + current_polygon_info = detection_info.road_polygon_info + polygon = current_polygon_info.polygon + + # if car_trajectory: + # for point in car_trajectory: + # if point.timestamp > current_time and not shapely.geometry.Polygon(polygon).contains( + # shapely.geometry.Point(point.coordinates[:2]) + # ): + # return point.timestamp, point.coordinates[:2] + # return None, None + if detection_info.road_type == "intersection" or ( + detection_info.segment_heading is None and detection_info.road_type != "intersection" + ): return None, None segmentheading = detection_info.segment_heading + 90 - car_loc = shapely.geometry.Point(car_loc[:2]) + _car_loc = shapely.geometry.Point(car_loc[:2]) car_vector = (math.cos(math.radians(segmentheading)), math.sin(math.radians(segmentheading))) - car_heading_point = (car_loc.x + car_vector[0], car_loc.y + car_vector[1]) - car_heading_line = shapely.geometry.LineString([car_loc, car_heading_point]) + car_heading_point = (_car_loc.x + car_vector[0], _car_loc.y + car_vector[1]) + # car_heading_line = shapely.geometry.LineString([_car_loc, car_heading_point]) + car_heading_line = ( + (float(_car_loc.x), float(_car_loc.y)), + (float(car_heading_point[0]), float(car_heading_point[1])), + ) intersection = line_to_polygon_intersection(polygon, car_heading_line) if len(intersection) == 2: - intersection_1_vector = (intersection[0][0] - car_loc.x, intersection[0][1] - car_loc.y) + intersection_1_vector = (intersection[0][0] - _car_loc.x, intersection[0][1] - _car_loc.y) relative_direction_1 = relative_direction(car_vector, intersection_1_vector) - intersection_2_vector = (intersection[1][0] - car_loc.x, intersection[1][1] - car_loc.y) + intersection_2_vector = (intersection[1][0] - _car_loc.x, intersection[1][1] - _car_loc.y) relative_direction_2 = relative_direction(car_vector, intersection_2_vector) - distance1 = compute_distance(car_loc, intersection[0]) - distance2 = compute_distance(car_loc, intersection[1]) + distance1 = compute_distance(_car_loc, intersection[0]) + distance2 = compute_distance(_car_loc, intersection[1]) if relative_direction_1: # logger.info(f'relative_dierction_1 {distance1} {current_time} {max_car_speed(current_polygon_info.road_type)}') return ( @@ -304,7 +310,7 @@ def get_car_exits_view_frame_num( detection_info: "DetectionInfo", ego_views: "list[postgis.Polygon]", max_frame_num: int, - fps=20, + fps: int | float = 20, ): car_heading = detection_info.segment_heading road_type = detection_info.road_type diff --git a/spatialyze/video_processor/stages/segment_trajectory/__init__.py b/spatialyze/video_processor/stages/segment_trajectory/__init__.py deleted file mode 100644 index b0e88a3..0000000 --- a/spatialyze/video_processor/stages/segment_trajectory/__init__.py +++ /dev/null @@ -1,94 +0,0 @@ -import datetime -from dataclasses import dataclass -from typing import Any, Dict, NamedTuple, Union - -import postgis -import shapely.geometry as sg - -from ...types import DetectionId, Float3 -from ..detection_estimation.segment_mapping import RoadPolygonInfo -from ..stage import Stage - - -class PolygonAndId(NamedTuple): - id: "str" - polygon: "sg.Polygon" - - -@dataclass -class ValidSegmentPoint: - detection_id: "DetectionId" - car_loc3d: "Float3" - timestamp: "datetime.datetime" - segment_type: "str" - segment_line: "postgis.LineString" - segment_heading: "float" - road_polygon_info: "RoadPolygonInfo | PolygonAndId" - obj_id: "int | None" = None - type: "str | None" = None - next: "SegmentPoint | None" = None - prev: "SegmentPoint | None" = None - - -@dataclass -class InvalidSegmentPoint: - detection_id: "DetectionId" - car_loc3d: "Float3" - timestamp: "datetime.datetime" - obj_id: "int | None" = None - type: "str | None" = None - next: "SegmentPoint | None" = None - prev: "SegmentPoint | None" = None - - -SegmentPoint = Union[ValidSegmentPoint, InvalidSegmentPoint] - - -SegmentTrajectoryMetadatum = Dict[int, SegmentPoint] - - -class SegmentTrajectory(Stage[SegmentTrajectoryMetadatum]): - @classmethod - def encode_json(cls, o: "Any"): - if isinstance(o, ValidSegmentPoint): - return { - "detection_id": tuple(o.detection_id), - "car_loc3d": o.car_loc3d, - "timestamp": str(o.timestamp), - "segment_line": None if o.segment_line is None else o.segment_line.to_ewkb(), - # "segment_line_wkb": o.segment_line.wkb_hex, - "segment_heading": o.segment_heading, - "road_polygon_info": o.road_polygon_info, - "obj_id": o.obj_id, - "type": o.type, - "next": None if o.next is None else tuple(o.next.detection_id), - "prev": None if o.prev is None else tuple(o.prev.detection_id), - } - if isinstance(o, InvalidSegmentPoint): - return { - "detection_id": tuple(o.detection_id), - "car_loc3d": o.car_loc3d, - "timestamp": str(o.timestamp), - "obj_id": o.obj_id, - "type": o.type, - "next": None if o.next is None else tuple(o.next.detection_id), - "prev": None if o.prev is None else tuple(o.prev.detection_id), - } - if isinstance(o, RoadPolygonInfo): - return { - "id": o.id, - "polygon": str(o.polygon), - # "polygon_wkb": o.polygon.wkb_hex, - "segment_lines": [*map(str, o.segment_lines)], - "road_type": o.road_type, - "segment_headings": o.segment_headings, - "contains_ego": o.contains_ego, - "ego_config": o.ego_config, - "fov_lines": o.fov_lines, - } - - if isinstance(o, PolygonAndId): - return { - "id": o.id, - "polygon": str(o.polygon), - } diff --git a/spatialyze/video_processor/stages/segment_trajectory/from_tracking_3d.py b/spatialyze/video_processor/stages/segment_trajectory/from_tracking_3d.py deleted file mode 100644 index 85fff3a..0000000 --- a/spatialyze/video_processor/stages/segment_trajectory/from_tracking_3d.py +++ /dev/null @@ -1,367 +0,0 @@ -import datetime -from typing import NamedTuple - -import numpy as np -import postgis -import psycopg2.sql -import shapely.geometry as sg -import shapely.wkb -import torch - -from ....database import database -from ...payload import Payload -from ...types import DetectionId, Float2 -from ..detection_3d import Detection3D -from ..detection_3d import Metadatum as Detection3DMetadatum -from ..tracking.tracking import Metadatum as TrackingMetadatum -from ..tracking.tracking import Tracking -from . import ( - InvalidSegmentPoint, - PolygonAndId, - SegmentPoint, - SegmentTrajectory, - SegmentTrajectoryMetadatum, - ValidSegmentPoint, -) - -USEFUL_TYPES = ["lane", "lanegroup", "intersection"] - - -class FromTracking3D(SegmentTrajectory): - def __init__(self): - self.analyze = True - self.explains = [] - - def _run(self, payload: "Payload"): - d3d: "list[Detection3DMetadatum] | None" = Detection3D.get(payload) - assert d3d is not None - - tracking: "list[TrackingMetadatum] | None" = Tracking.get(payload) - assert tracking is not None - - class_map: "list[str] | None" = d3d[0].class_map - assert class_map is not None - - # Index detections using their detection id - detection_map: "dict[DetectionId, tuple[int, int]]" = dict() - for fidx, (_, names, dids) in enumerate(d3d): - for oidx, did in enumerate(dids): - assert did not in detection_map - detection_map[did] = (fidx, oidx) - - object_map: "dict[int, dict[DetectionId, torch.Tensor]]" = dict() - for fidx, frame in enumerate(tracking): - for tracking_result in frame: - did = tracking_result.detection_id - oid = tracking_result.object_id - if oid not in object_map: - object_map[oid] = {} - if did in object_map[oid]: - continue - fidx, oidx = detection_map[did] - detections, *_ = d3d[fidx] - object_map[oid][did] = detections[oidx] - - # Index object trajectories using their object id - # object_map: "dict[int, list[Tracking3DResult]]" = dict() - # for frame in t3d: - # for oid, t in frame.items(): - # if oid not in object_map: - # object_map[oid] = [] - - # object_map[oid].append(t) - - # Create a list of detection points with each of its corresponding direction - points: "list[tuple[tuple[DetectionId, torch.Tensor], tuple[float, float] | None]]" = [] - for traj_dict in object_map.values(): - traj = [*traj_dict.items()] - traj.sort(key=lambda t: t[0].frame_idx) - - if len(traj) <= 1: - points.extend((t, None) for t in traj) - continue - - # First and last points' direction - points.append((traj[0], _get_direction_2d(traj[0], traj[1]))) - points.append((traj[-1], _get_direction_2d(traj[-2], traj[-1]))) - - # All other points' direction - for prv, cur, nxt in zip(traj[:-2], traj[1:-1], traj[2:]): - points.append((cur, _get_direction_2d(prv, nxt))) - - locations = set(f.location for f in payload.video._camera_configs) - assert len(locations) == 1, locations - - location = [*locations][0] - - # Map a segment to each detection - # Note: Some detection might be missing due to not having any segment mapped - segments = map_points_and_directions_to_segment( - points, location, payload.video.videofile, self.explains - ) - - # Index segments using their detection id - segment_map: "dict[DetectionId, SegmentMapping]" = {} - for segment in segments: - did = DetectionId(*segment[:2]) - if did not in segment_map: - # assert did not in segment_map - segment_map[did] = segment - - object_id_to_segmnt_map: "dict[int, list[SegmentPoint]]" = {} - output: "list[SegmentTrajectoryMetadatum]" = [dict() for _ in range(len(payload.video))] - for oid, obj in object_map.items(): - segment_trajectory: "list[SegmentPoint]" = [] - object_id_to_segmnt_map[oid] = segment_trajectory - - for did, det in obj.items(): - # did = det.detection_id - timestamp = payload.video.interpolated_frames[did.frame_idx].timestamp - args = did, timestamp, det, oid, class_map - if did in segment_map: - # Detection that can be mapped to a segment - segment = segment_map[did] - _fid, _oid, polygonid, polygon, segmentid, types, line, heading = segment - assert did.frame_idx == _fid - assert did.obj_order == _oid - - polygon = shapely.wkb.loads(polygon.to_ewkb(), hex=True) - assert isinstance(polygon, sg.Polygon) - - type_ = next((t for t in types if t in USEFUL_TYPES), types[-1]) - - segment_point = valid_segment_point( - *args, type_, line, heading, polygonid, polygon - ) - else: - # Detection that cannot be mapped to any segment - segment_point = invalid_segment_point(*args) - - segment_trajectory.append(segment_point) - - metadatum = output[did.frame_idx] - if oid in metadatum: - assert metadatum[oid].detection_id == segment_point.detection_id - if metadatum[oid].timestamp > segment_point.timestamp: - metadatum[oid] = segment_point - metadatum[oid] = segment_point - - for prv, nxt in zip(segment_trajectory[:-1], segment_trajectory[1:]): - prv.next = nxt - nxt.prev = prv - - return None, {self.classname(): output} - - -def invalid_segment_point( - did: "DetectionId", - timestamp: "datetime.datetime", - det: "torch.Tensor", - oid: "int", - class_map: "list[str]", -): - x, y, z = map(float, ((det[6:9] + det[9:12]) / 2).tolist()) - return InvalidSegmentPoint( - did, - (x, y, z), - timestamp, - oid, - class_map[int(det[5])], - ) - - -def valid_segment_point( - did: "DetectionId", - timestamp: "datetime.datetime", - det: "torch.Tensor", - oid: "int", - class_map: "list[str]", - segmenttype: "str", - segmentline: "postgis.LineString", - segmentheading: "float", - polygonid: "str", - shapely_polygon: "sg.Polygon", -): - x, y, z = map(float, ((det[6:9] + det[9:12]) / 2).tolist()) - return ValidSegmentPoint( - did, - (x, y, z), - timestamp, - segmenttype, - segmentline, - segmentheading, - # A place-holder for Polygon that only contain polygon id and polygon - PolygonAndId(polygonid, shapely_polygon), - oid, - class_map[int(det[5])], - ) - - -def _get_direction_2d( - p1: "tuple[DetectionId, torch.Tensor]", p2: "tuple[DetectionId, torch.Tensor]" -) -> "Float2": - _p1 = (p1[1][6:9] + p1[1][9:12]) / 2 - _p2 = (p2[1][6:9] + p2[1][9:12]) / 2 - diff = (_p2 - _p1)[:2].cpu() - udiff = diff / np.linalg.norm(diff) - x, y = map(float, udiff.numpy()) - return x, y - - -class SegmentMapping(NamedTuple): - fid: "int" - oid: "int" - elementid: "str" - polygon: "postgis.Polygon" - segmentid: "int" - segmenttypes: "list[str]" - line: "postgis.LineString" - heading: "float" - - -# TODO: should we try to map points to closest segment instead of just ignoring them? - - -def map_points_and_directions_to_segment( - annotations: "list[tuple[tuple[DetectionId, torch.Tensor], tuple[float, float] | None]]", - # annotations: "list[tuple[Tracking3DResult, tuple[float, float] | None]]", - location: "str", - videofile: "str", - explains: "list[dict]", -) -> "list[SegmentMapping]": - if len(annotations) == 0: - return [] - - frame_indices = [a[0].frame_idx for a, _ in annotations] - object_indices = [a[0].obj_order for a, _ in annotations] - points = [(a[1][6:9] + a[1][9:12]) / 2 for a, _ in annotations] - txs = [float(p[0].item()) for p in points] - tys = [float(p[1].item()) for p in points] - dxs = [d and d[0] for _, d in annotations] - dys = [d and d[1] for _, d in annotations] - - _point = psycopg2.sql.SQL( - "UNNEST({fid}, {oid}, {tx}, {ty}, {dx}::double precision[], {dy}::double precision[]) AS _point (fid, oid, tx, ty, dx, dy)" - ).format( - fid=psycopg2.sql.Literal(frame_indices), - oid=psycopg2.sql.Literal(object_indices), - tx=psycopg2.sql.Literal(txs), - ty=psycopg2.sql.Literal(tys), - dx=psycopg2.sql.Literal(dxs), - dy=psycopg2.sql.Literal(dys), - # fields=psycopg2.sql.SQL(',').join(map(psycopg2.sql.Literal, [frame_indices, object_indices, txs, tys, dxs, dys])) - ) - - helper = psycopg2.sql.SQL( - """ - SET client_min_messages TO WARNING; - DROP FUNCTION IF EXISTS _angle(double precision); - CREATE OR REPLACE FUNCTION _angle(a double precision) RETURNS double precision AS - $BODY$ - BEGIN - RETURN ((a::decimal % 360) + 360) % 360; - END - $BODY$ - LANGUAGE 'plpgsql'; - """ - ) - - query = psycopg2.sql.SQL( - """ - WITH - Point AS (SELECT * FROM {_point}), - AvailablePolygon AS ( - SELECT * - FROM SegmentPolygon - WHERE location = {location} - AND (SegmentPolygon.__RoadType__intersection__ - OR SegmentPolygon.__RoadType__lane__ - OR SegmentPolygon.__RoadType__lanegroup__ - OR SegmentPolygon.__RoadType__lanesection__) - ), - _SegmentWithDirection AS ( - SELECT - *, - ST_X(endPoint) - ST_X(startPoint) AS _x, - ST_Y(endPoint) - ST_Y(startPoint) AS _y - FROM Segment - ), - SegmentWithDirection AS ( - SELECT - *, - (_x / SQRT(POWER(_x, 2) + POWER(_y, 2))) AS dx, - (_y / SQRT(POWER(_x, 2) + POWER(_y, 2))) AS dy - FROM _SegmentWithDirection - WHERE - _x <> 0 OR _y <> 0 - ), - MinPolygon AS ( - SELECT fid, oid, MIN(ST_Area(Polygon.elementPolygon)) as size - FROM Point AS p - JOIN AvailablePolygon AS Polygon - ON ST_Contains(Polygon.elementPolygon, ST_Point(p.tx, p.ty)) - GROUP BY fid, oid - ), - MinPolygonId AS ( - SELECT fid, oid, MIN(elementId) as elementId - FROM Point AS p - JOIN MinPolygon USING (fid, oid) - JOIN AvailablePolygon as Polygon - ON ST_Contains(Polygon.elementPolygon, ST_Point(p.tx, p.ty)) - AND ST_Area(Polygon.elementPolygon) = MinPolygon.size - GROUP BY fid, oid - ), - PointPolygonSegment AS ( - SELECT - *, - ST_Distance(ST_Point(tx, ty), ST_MakeLine(startPoint, endPoint)) AS distance, - CASE - WHEN p.dx IS NULL THEN 0 - WHEN p.dy IS NULL THEN 0 - ELSE _angle(ACOS((p.dx * sd.dx) + (p.dy * sd.dy)) * 180 / PI()) - END AS anglediff - FROM Point AS p - JOIN MinPolygonId USING (fid, oid) - JOIN AvailablePolygon USING (elementId) - JOIN SegmentWithDirection AS sd USING (elementId) - WHERE - AvailablePolygon.__RoadType__intersection__ - OR - p.dx IS NULL - OR - p.dy IS NULL - OR - _angle(ACOS((p.dx * sd.dx) + (p.dy * sd.dy)) * 180 / PI()) < 90 - OR - _angle(ACOS((p.dx * sd.dx) + (p.dy * sd.dy)) * 180 / PI()) > 270 - ), - MinDis as ( - SELECT fid, oid, MIN(distance) as mindistance - FROM PointPolygonSegment - GROUP BY fid, oid - ), - MinDisMinAngle as ( - SELECT fid, oid, MIN(LEAST(pps.anglediff, 360-pps.anglediff)) as minangle - FROM PointPolygonSegment AS pps - JOIN MinDis USING (fid, oid) - WHERE pps.distance = MinDis.mindistance - GROUP BY fid, oid - ) - - SELECT fid, oid, elementid, elementpolygon, segmentid, segmenttypes, segmentline, heading - FROM PointPolygonSegment - JOIN MinDis USING (fid, oid) - JOIN MinDisMinAngle USING (fid, oid) - WHERE PointPolygonSegment.distance = MinDis.mindistance - AND PointPolygonSegment.anglediff = MinDisMinAngle.minangle - """ - ).format(_point=_point, location=psycopg2.sql.Literal(location)) - - # explain = psycopg2.sql.SQL(" EXPLAIN (ANALYZE, COSTS, VERBOSE, BUFFERS, FORMAT JSON) ") - # explains.append({ - # 'name': videofile, - # 'analyze': database.execute(helper + explain + query)[0][0][0] - # }) - - result = database.execute(helper + query) - return list(map(SegmentMapping._make, result)) diff --git a/spatialyze/video_processor/types.py b/spatialyze/video_processor/types.py index 81a6565..27b3f79 100644 --- a/spatialyze/video_processor/types.py +++ b/spatialyze/video_processor/types.py @@ -1,8 +1,17 @@ -from typing import NamedTuple, Tuple +from typing import NamedTuple UNIQUE_MAP: dict[int, int] = {} +Float2 = tuple[float, float] +Float22 = tuple[Float2, Float2] + +Float3 = tuple[float, float, float] +Float33 = tuple[Float3, Float3, Float3] + +Float4 = tuple[float, float, float, float] + + class DetectionId(NamedTuple): frame_idx: int obj_order: "str | int" @@ -20,16 +29,7 @@ def unique(cls, frame_idx: int) -> int: class obj_detection(NamedTuple): detection_id: DetectionId - car_loc3d: "Float3" - car_loc2d: "Float2" - car_bbox3d: "Tuple[Float3, Float3]" - car_bbox2d: "Float22" - - -Float2 = Tuple[float, float] -Float22 = Tuple[Float2, Float2] - -Float3 = Tuple[float, float, float] -Float33 = Tuple[Float3, Float3, Float3] - -Float4 = Tuple[float, float, float, float] + car_loc3d: Float3 + car_loc2d: Float2 + car_bbox3d: tuple[Float3, Float3] + car_bbox2d: Float22 diff --git a/tests/interface/utils/F/common.py b/tests/interface/utils/F/common.py index 1b7b70d..e248b5a 100644 --- a/tests/interface/utils/F/common.py +++ b/tests/interface/utils/F/common.py @@ -1,5 +1,6 @@ from spatialyze.predicate import GenSqlVisitor, objects, camera from spatialyze.utils.F import * +from spatialyze.utils.F.custom_fn import custom_fn o = objects[0] diff --git a/tests/interface/utils/F/test_custom_fn.py b/tests/interface/utils/F/test_custom_fn.py new file mode 100644 index 0000000..82d587f --- /dev/null +++ b/tests/interface/utils/F/test_custom_fn.py @@ -0,0 +1,20 @@ +import pytest +from common import * + + +@pytest.mark.parametrize("fn, sql", [ + (custom_fn('c1', 1)(1), "c1(1)"), + (custom_fn('c2', 2)(1, 2), "c2(1,2)"), + (custom_fn('c3', 3)(1, 2, 3), "c3(1,2,3)"), +]) +def test_distance(fn, sql): + assert gen(fn) == sql + + +@pytest.mark.parametrize("fn, msg", [ + (custom_fn('c1', 1)(o, o), "c1 is expecting 1 arguments, but received 2"), +]) +def test_exception(fn, msg): + with pytest.raises(Exception) as e_info: + gen(fn) + assert str(e_info.value) == msg diff --git a/tests/interface/utils/F/test_same_region.py b/tests/interface/utils/F/test_same_region.py new file mode 100644 index 0000000..ea4ca48 --- /dev/null +++ b/tests/interface/utils/F/test_same_region.py @@ -0,0 +1,23 @@ +import pytest +from common import * + + +o = objects[0] +c = camera + + +@pytest.mark.parametrize("fn, sql", [ + (same_region('intersection', o, c), "sameRegion('intersection',valueAtTimestamp(t0.translations,c0.timestamp),c0.cameraTranslation)"), +]) +def test_same_retion(fn, sql): + assert gen(fn) == sql + + +@pytest.mark.parametrize("fn, msg", [ + (same_region(c, o, o), "Unsupported road type: CameraTableNode[0]"), + (same_region('invalid', o, o), "Unsupported road type: LiteralNode(value='invalid', python=True)"), +]) +def test_exception(fn, msg): + with pytest.raises(Exception) as e_info: + gen(fn) + assert str(e_info.value) == msg \ No newline at end of file diff --git a/tests/video_processor/steps/test_detection_2d.py b/tests/video_processor/steps/test_detection_2d.py index ea6b484..f54040c 100644 --- a/tests/video_processor/steps/test_detection_2d.py +++ b/tests/video_processor/steps/test_detection_2d.py @@ -50,7 +50,7 @@ def test_detection_2d(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) keep = bitarray(len(frames)) keep.setall(0) @@ -90,7 +90,7 @@ def test_groundtruth(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) keep = bitarray(len(frames)) keep.setall(0) diff --git a/tests/video_processor/steps/test_detection_estimation.py b/tests/video_processor/steps/test_detection_estimation.py new file mode 100644 index 0000000..cddab81 --- /dev/null +++ b/tests/video_processor/steps/test_detection_estimation.py @@ -0,0 +1,55 @@ +import os +import pickle +from bitarray import bitarray +import json +import pickle + +from spatialyze.video_processor.pipeline import Pipeline +from spatialyze.video_processor.payload import Payload +from spatialyze.video_processor.stages.detection_3d.from_detection_2d_and_road import FromDetection2DAndRoad +from spatialyze.video_processor.video import Video +from spatialyze.video_processor.camera_config import camera_config +from spatialyze.video_processor.stages.detection_estimation import DetectionEstimation + +from common import yolo_output + +OUTPUT_DIR = './data/pipeline/test-results' +VIDEO_DIR = './data/pipeline/videos' + + +def test_detection_estimation(): + files = os.listdir(VIDEO_DIR) + + with open(os.path.join(VIDEO_DIR, 'frames.pkl'), 'rb') as f: + videos = pickle.load(f) + + pipeline = Pipeline([ + # Manually ingest processed detections from YoloDetection + # DecodeFrame(), + # YoloDetection(), + FromDetection2DAndRoad(), + DetectionEstimation(), + ]) + + for name, video in videos.items(): + if video['filename'] not in files: + continue + + frames = Video( + os.path.join(VIDEO_DIR, video["filename"]), + [camera_config(*f) for f in video["frames"]], + ) + keep = bitarray(len(frames)) + keep.setall(0) + keep[(len(frames) * 7) // 8:] = 1 + + output = pipeline.run(Payload(frames, keep, metadata=yolo_output(name))) + keep_result = [*output.keep] + if os.environ.get('GENERATE_PROCESSOR_TEST_RESULTS', 'false') == 'true': + with open(os.path.join(OUTPUT_DIR, f'DetectionEstimationKeep--{name}.json'), 'w') as f: + json.dump(keep_result, f, indent=1) + + with open(os.path.join(OUTPUT_DIR, f'DetectionEstimationKeep--{name}.json'), 'r') as f: + keep_groundtruth = json.load(f) + + assert keep_result == keep_groundtruth diff --git a/tests/video_processor/steps/test_in_view.py b/tests/video_processor/steps/test_in_view.py index 615542d..c1ba907 100644 --- a/tests/video_processor/steps/test_in_view.py +++ b/tests/video_processor/steps/test_in_view.py @@ -99,6 +99,8 @@ def test_predicates(fn, sqls): node = KeepOnlyRoadTypePredicates()(fn) assert gen(node) == sqls[0], node + node1 = None + node2 = None if len(sqls) > 1: sql = sqls[1] @@ -107,12 +109,14 @@ def test_predicates(fn, sqls): if len(sqls) > 2: sql = sqls[2] + assert node1 is not None node2 = NormalizeInversionAndFlattenRoadTypePredicates()(node1) assert gen(node2) == (gen(node1) if sql is None else sql), node2 if len(sqls) > 3: assert isinstance(sqls[3], str), sqls[3] assert isinstance(sqls[4], set), sqls[4] + assert node2 is not None predicate_str = InViewPredicate(RT)(node2) assert predicate_str == sqls[3], predicate_str @@ -196,7 +200,7 @@ def test_detection_2d(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) output1 = pipeline1.run(Payload(frames)) diff --git a/tests/video_processor/steps/test_objecttype.py b/tests/video_processor/steps/test_objecttype.py index 9c16018..d4cf160 100644 --- a/tests/video_processor/steps/test_objecttype.py +++ b/tests/video_processor/steps/test_objecttype.py @@ -82,7 +82,7 @@ def test_filter(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) keep = bitarray(len(frames)) keep.setall(0) diff --git a/tests/video_processor/steps/test_tracking_2d.py b/tests/video_processor/steps/test_tracking_2d.py index a731a8f..eddf9b2 100644 --- a/tests/video_processor/steps/test_tracking_2d.py +++ b/tests/video_processor/steps/test_tracking_2d.py @@ -58,7 +58,7 @@ def test_strongsort(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) keep = bitarray(len(frames)) keep.setall(0) @@ -97,7 +97,7 @@ def test_deepsort(): frames = Video( os.path.join(VIDEO_DIR, video["filename"]), - [camera_config(*f, 0) for f in video["frames"]], + [camera_config(*f) for f in video["frames"]], ) keep = bitarray(len(frames)) keep.setall(0)