From affecc21cefeb17f220eda7545ec84dc25fc12d7 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:47:59 +0100 Subject: [PATCH 001/107] Add pygeos package --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f77752677..6004f125c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ opencv-python==4.8.1.78 openpyxl==3.1.2 pandas==2.1.1 pillow==10.1.0 +pygeos==0.14 seaborn==0.13.0 shapely==2.0.2 tqdm==4.66.1 From bfd2cdcf35c72bfe2ea9b593a0ae6af74428db0b Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:39:38 +0100 Subject: [PATCH 002/107] Extend TrackDataset interface with methods to get tracks intersecting sections and intersection points of tracks and sections --- OTAnalytics/domain/track.py | 15 +++++++++++++++ .../plugin_datastore/python_track_store.py | 11 ++++++++++- OTAnalytics/plugin_datastore/track_store.py | 11 ++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index c4cd8a100..31dc15e00 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -9,6 +9,7 @@ from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.observer import Subject +from OTAnalytics.domain.section import Section, SectionId MIN_NUMBER_OF_DETECTIONS = 5 CLASSIFICATION: str = "classification" @@ -364,6 +365,9 @@ def __init__(self, track_ids: list[TrackId], message: str): self._track_ids = track_ids +INTERSECTION_COORDINATE = tuple[float, float] + + class TrackDataset(ABC): def __iter__(self) -> Iterator[Track]: yield from self.as_list() @@ -404,6 +408,17 @@ def clear(self) -> "TrackDataset": def as_list(self) -> list[Track]: raise NotImplementedError + @abstractmethod + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + raise NotImplementedError + + @abstractmethod + def intersection_points( + self, + sections: list[Section], + ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + raise NotImplementedError + @abstractmethod def split(self, chunks: int) -> Sequence["TrackDataset"]: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 642cd522c..0fe1cc7fb 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -7,7 +7,9 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain.common import DataclassValidation +from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( + INTERSECTION_COORDINATE, Detection, Track, TrackClassificationCalculator, @@ -210,7 +212,6 @@ def calculate(self, detections: list[Detection]) -> str: return max(classifications, key=lambda x: classifications[x]) -@dataclass class PythonTrackDataset(TrackDataset): """Pure Python implementation of a TrackDataset.""" @@ -309,3 +310,11 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": if len(track.detections) >= length } return PythonTrackDataset(filtered_tracks, self._calculator) + + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + raise NotImplementedError + + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 957db70a6..a09a17c23 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -9,7 +9,9 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track +from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( + INTERSECTION_COORDINATE, MIN_NUMBER_OF_DETECTIONS, Detection, Track, @@ -138,7 +140,6 @@ def calculate(self, detections: DataFrame) -> DataFrame: DEFAULT_CLASSIFICATOR = PandasByMaxConfidence() -@dataclass class PandasTrackDataset(TrackDataset): def __init__( self, @@ -249,6 +250,14 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": ] return PandasTrackDataset(filtered_dataset, self._calculator) + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + raise NotImplementedError + + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + raise NotImplementedError + def _assign_track_classification( data: DataFrame, calculator: PandasTrackClassificationCalculator From 67bff15c018014e3bf31ac8106b2c3accdd0b593 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:44:48 +0100 Subject: [PATCH 003/107] Store tracks in TrackDataset in separate storage as pygeos geometries --- OTAnalytics/plugin_datastore/track_store.py | 49 +++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index a09a17c23..c0d493b16 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -7,6 +7,11 @@ import pandas from more_itertools import batched from pandas import DataFrame, Series +from pygeos import get_coordinates as pygeos_coords +from pygeos import line_locate_point as pygeos_project +from pygeos import linestrings +from pygeos import points as pygeos_points +from pygeos import prepare from OTAnalytics.domain import track from OTAnalytics.domain.section import Section, SectionId @@ -147,8 +152,28 @@ def __init__( calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ): self._dataset = dataset + self._track_geom_df = self._create_track_geom_df(self._dataset) self._calculator = calculator + def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: + if dataset.empty: + return DataFrame(columns=["id", "geom"]) + track_geom_df = DataFrame.from_records( + [ + (track_id, linestrings(list(zip(detections.x, detections.y)))) + for track_id, detections in dataset.groupby(track.TRACK_ID) + ], + columns=["id", "geom"], + ) + track_geom_df["projection"] = track_geom_df["geom"].apply( + lambda track_geom: [ + pygeos_project(track_geom, pygeos_points(p)) + for p in pygeos_coords(track_geom) + ] + ) + track_geom_df["geom"].apply(prepare) + return track_geom_df + @staticmethod def from_list( tracks: list[Track], @@ -256,6 +281,30 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + # sec_geoms = sections_to_pygeos_multi( + # sections + # ) # [self.section_geom_map[section.id] for section in sections] + # df["intersections"] = df["geom"].apply( + # lambda line: [ + # i for i in pygeos_intersection(line, sec_geoms) if not is_empty(i) + # ] + # ) + # + # vs = ( + # df[df["intersections"].apply(lambda i: len(i) > 0)] + # .apply( + # lambda r: [ + # self.next_event( + # r["track"], r["geom"], pygeos_points(p), r["projection"] + # ) + # for p in pygeos_coords(r["intersections"]) + # ], + # axis=1, + # ) + # .values + # ) + # + # return [v for list in vs for v in list] raise NotImplementedError From 6c59153debaca13b3fb8cc877cd5efe98f31c9a7 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:26:42 +0100 Subject: [PATCH 004/107] Order saved track geometries by offset in track dataset --- OTAnalytics/plugin_datastore/track_store.py | 22 +++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index c0d493b16..bf61645f8 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -13,7 +13,9 @@ from pygeos import points as pygeos_points from pygeos import prepare +from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( INTERSECTION_COORDINATE, @@ -150,9 +152,13 @@ def __init__( self, dataset: DataFrame = DataFrame(), calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, + default_track_offset: RelativeOffsetCoordinate = DEFAULT_TRACK_OFFSET, ): self._dataset = dataset - self._track_geom_df = self._create_track_geom_df(self._dataset) + self._default_track_offset = default_track_offset + self._track_geometries = { + self._default_track_offset: self._create_track_geom_df(self._dataset) + } self._calculator = calculator def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: @@ -160,7 +166,19 @@ def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: return DataFrame(columns=["id", "geom"]) track_geom_df = DataFrame.from_records( [ - (track_id, linestrings(list(zip(detections.x, detections.y)))) + ( + track_id, + linestrings( + list( + zip( + detections.x + + detections.x * self._default_track_offset.x, + detections.y + + detections.y * self._default_track_offset.y, + ) + ) + ), + ) for track_id, detections in dataset.groupby(track.TRACK_ID) ], columns=["id", "geom"], From 3bd4595748db41d514762a180a2f1172a66afeff Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:55:08 +0100 Subject: [PATCH 005/107] Move group_sections_by_offset function --- OTAnalytics/application/analysis/intersect.py | 15 ++++++++++++++- .../shapely/create_intersection_events.py | 18 +++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index bf18739ed..a3bf969f3 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -1,9 +1,12 @@ from abc import ABC, abstractmethod -from typing import Iterable +from collections import defaultdict +from typing import Iterable, Mapping from OTAnalytics.domain.event import Event +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.types import EventType class RunIntersect(ABC): @@ -22,3 +25,13 @@ class TracksIntersectingSections(ABC): @abstractmethod def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: raise NotImplementedError + + +def group_sections_by_offset( + sections: Iterable[Section], +) -> Mapping[RelativeOffsetCoordinate, Iterable[Section]]: + grouped_sections: dict[RelativeOffsetCoordinate, list[Section]] = defaultdict(list) + for section in sections: + offset = section.get_offset(EventType.SECTION_ENTER) + grouped_sections[offset].append(section) + return grouped_sections diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index 25e4f7da6..d9749a27b 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -1,10 +1,12 @@ -from collections import defaultdict from functools import singledispatchmethod -from typing import Callable, Iterable, Mapping +from typing import Callable, Iterable from shapely import LineString, Polygon, contains_xy, prepare -from OTAnalytics.application.analysis.intersect import RunIntersect +from OTAnalytics.application.analysis.intersect import ( + RunIntersect, + group_sections_by_offset, +) from OTAnalytics.application.geometry import GeometryBuilder from OTAnalytics.application.use_cases.track_repository import ( GetTracksWithoutSingleDetections, @@ -293,13 +295,3 @@ def _create_events(tracks: Iterable[Track], sections: Iterable[Section]) -> list ) events.extend(create_intersection_events.create()) return events - - -def group_sections_by_offset( - sections: Iterable[Section], -) -> Mapping[RelativeOffsetCoordinate, Iterable[Section]]: - grouped_sections: dict[RelativeOffsetCoordinate, list[Section]] = defaultdict(list) - for section in sections: - offset = section.get_offset(EventType.SECTION_ENTER) - grouped_sections[offset].append(section) - return grouped_sections From 945f69092c63a2fd0c931908eee0fed8d16a9f3e Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 10:54:33 +0100 Subject: [PATCH 006/107] Add getting intersection points from TrackDataset with sections --- OTAnalytics/application/analysis/intersect.py | 4 +- OTAnalytics/domain/track.py | 6 +- .../plugin_datastore/python_track_store.py | 4 +- OTAnalytics/plugin_datastore/track_store.py | 114 ++++++++--- .../plugin_datastore/test_track_store.py | 193 +++++++++++++++++- 5 files changed, 278 insertions(+), 43 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index a3bf969f3..61c9b69c7 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from collections import defaultdict -from typing import Iterable, Mapping +from typing import Iterable, Mapping, Sequence from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import RelativeOffsetCoordinate @@ -29,7 +29,7 @@ def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]] def group_sections_by_offset( sections: Iterable[Section], -) -> Mapping[RelativeOffsetCoordinate, Iterable[Section]]: +) -> Mapping[RelativeOffsetCoordinate, Sequence[Section]]: grouped_sections: dict[RelativeOffsetCoordinate, list[Section]] = defaultdict(list) for section in sections: offset = section.get_offset(EventType.SECTION_ENTER) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 31dc15e00..b837bf3d3 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -365,7 +365,9 @@ def __init__(self, track_ids: list[TrackId], message: str): self._track_ids = track_ids -INTERSECTION_COORDINATE = tuple[float, float] +@dataclass +class IntersectionPoint: + index: int class TrackDataset(ABC): @@ -416,7 +418,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section], - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError @abstractmethod diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 0fe1cc7fb..dc75e2fa9 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -9,8 +9,8 @@ from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - INTERSECTION_COORDINATE, Detection, + IntersectionPoint, Track, TrackClassificationCalculator, TrackDataset, @@ -316,5 +316,5 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index bf61645f8..b8e37897d 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -1,26 +1,33 @@ from abc import ABC, abstractmethod +from bisect import bisect +from collections import defaultdict from dataclasses import dataclass from datetime import datetime +from itertools import chain from math import ceil from typing import Any, Iterable, Optional, Sequence import pandas from more_itertools import batched from pandas import DataFrame, Series +from pygeos import Geometry, geometrycollections from pygeos import get_coordinates as pygeos_coords +from pygeos import intersection as pygeos_intersection +from pygeos import is_empty from pygeos import line_locate_point as pygeos_project from pygeos import linestrings from pygeos import points as pygeos_points from pygeos import prepare +from OTAnalytics.application.analysis.intersect import group_sections_by_offset from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - INTERSECTION_COORDINATE, MIN_NUMBER_OF_DETECTIONS, Detection, + IntersectionPoint, Track, TrackDataset, TrackId, @@ -157,11 +164,16 @@ def __init__( self._dataset = dataset self._default_track_offset = default_track_offset self._track_geometries = { - self._default_track_offset: self._create_track_geom_df(self._dataset) + self._default_track_offset: self._create_track_geom_df( + self._dataset, self._default_track_offset + ) } self._calculator = calculator - def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: + @staticmethod + def _create_track_geom_df( + dataset: DataFrame, offset: RelativeOffsetCoordinate + ) -> DataFrame: if dataset.empty: return DataFrame(columns=["id", "geom"]) track_geom_df = DataFrame.from_records( @@ -171,10 +183,8 @@ def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: linestrings( list( zip( - detections.x - + detections.x * self._default_track_offset.x, - detections.y - + detections.y * self._default_track_offset.y, + detections.x + detections.x * offset.x, + detections.y + detections.y * offset.y, ) ) ), @@ -298,32 +308,62 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: - # sec_geoms = sections_to_pygeos_multi( - # sections - # ) # [self.section_geom_map[section.id] for section in sections] - # df["intersections"] = df["geom"].apply( - # lambda line: [ - # i for i in pygeos_intersection(line, sec_geoms) if not is_empty(i) - # ] - # ) - # - # vs = ( - # df[df["intersections"].apply(lambda i: len(i) > 0)] - # .apply( - # lambda r: [ - # self.next_event( - # r["track"], r["geom"], pygeos_points(p), r["projection"] - # ) - # for p in pygeos_coords(r["intersections"]) - # ], - # axis=1, - # ) - # .values - # ) - # - # return [v for list in vs for v in list] - raise NotImplementedError + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + sections_grouped_by_offset = group_sections_by_offset(sections) + + intersection_points = defaultdict(list) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + if (track_df := self._track_geometries.get(offset, None)) is None: + track_df = self._create_track_geom_df(self._dataset, offset) + self._track_geometries[offset] = track_df + + track_df["intersections"] = track_df["geom"].apply( + lambda line: [ + (_sections[index].id, ip) + for index, ip in enumerate(pygeos_intersection(line, section_geoms)) + if not is_empty(ip) + ] + ) + intersections = ( + track_df[track_df["intersections"].apply(lambda i: len(i) > 0)] + .apply( + lambda r: [ + self._next_event( + r["id"], + _section_id, + r["geom"], + pygeos_points(p), + r["projection"], + ) + for _section_id, ip in r["intersections"] + for p in pygeos_coords(ip) + ], + axis=1, + ) + .values + ) + for _id, section_id, intersection_point in chain.from_iterable( + intersections + ): + intersection_points[_id].append((section_id, intersection_point)) + + return intersection_points + + def _next_event( + self, + track_id: str, + section_id: SectionId, + track_geom: Geometry, + point: Geometry, + projection: Any, + ) -> tuple[TrackId, SectionId, IntersectionPoint]: + dist = pygeos_project(track_geom, point) + return ( + TrackId(track_id), + section_id, + IntersectionPoint(bisect(projection, dist)), + ) def _assign_track_classification( @@ -372,3 +412,11 @@ def _sort_tracks(track_df: DataFrame) -> DataFrame: return track_df.sort_values([track.TRACK_ID, track.FRAME]) else: return track_df + + +def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: + return geometrycollections([section_to_pygeos(s) for s in sections]) + + +def section_to_pygeos(section: Section) -> Geometry: + return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 1acf08663..b2f25f630 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -2,7 +2,10 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import LineSection, Section, SectionId +from OTAnalytics.domain.track import IntersectionPoint, Track, TrackDataset, TrackId +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasDetection, @@ -16,6 +19,154 @@ ) +@pytest.fixture +def first_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("1") + track_builder.add_xy_bbox(1, 1) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def second_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("2") + track_builder.add_xy_bbox(1, 1.5) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1.5) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1.5) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1.5) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1.5) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("3") + track_builder.add_xy_bbox(1, 10) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 10) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 10) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 10) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 10) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_section() -> Section: + name = "first" + coordinates = [Coordinate(0, 0), Coordinate(0, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def first_section() -> Section: + name = "first" + coordinates = [Coordinate(1.5, 0), Coordinate(1.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def second_section() -> Section: + name = "second" + coordinates = [Coordinate(2.5, 0), Coordinate(2.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def third_section() -> Section: + name = "third" + coordinates = [Coordinate(3.5, 0), Coordinate(3.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + class TestPandasDetection: def test_properties(self) -> None: builder = TrackBuilder() @@ -83,14 +234,15 @@ def test_add(self) -> None: builder.append_detection() builder.append_detection() track = builder.build_track() - detections = builder.build_serialized_detections() - expected_dataset = PandasTrackDataset(DataFrame(detections)) + expected_dataset = PandasTrackDataset.from_list([track]) dataset = PandasTrackDataset() merged = dataset.add_all(PythonTrackDataset({track.id: track})) assert 0 == len(dataset.as_list()) - assert merged == expected_dataset + for actual, expected in zip(merged, expected_dataset): + assert_equal_track_properties(actual, expected) + # assert merged == expected_dataset def test_add_nothing(self) -> None: dataset = PandasTrackDataset() @@ -209,3 +361,36 @@ def test_filter_by_minimum_detection_length(self) -> None: assert len(filtered_dataset) == 1 for actual_track, expected_track in zip(filtered_dataset, [second_track]): assert_equal_track_properties(actual_track, expected_track) + + def test_intersection_points( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + dataset = PandasTrackDataset.from_list( + [not_intersecting_track, first_track, second_track] + ) + result = dataset.intersection_points(list(sections)) + assert result == { + first_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + second_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + } From 04dc3cc4e1d653d99496f95bde83fee9c811caaa Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:03:43 +0100 Subject: [PATCH 007/107] Add documentation --- OTAnalytics/domain/track.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index b837bf3d3..f8c397722 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -412,6 +412,14 @@ def as_list(self) -> list[Track]: @abstractmethod def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ raise NotImplementedError @abstractmethod @@ -419,6 +427,16 @@ def intersection_points( self, sections: list[Section], ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """ + Return the intersection points resulting from the tracks and the + given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId]]]: the intersection points. + """ raise NotImplementedError @abstractmethod From 3e080725ab28aaac097a8c14a3804902a5da61cd Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:06:33 +0100 Subject: [PATCH 008/107] Extend TrackDataset with method to get boolean mask of track coordinates contained by sections This method will be required when trying to create intersection events with area sections. --- OTAnalytics/domain/track.py | 18 ++++++++++++++++-- .../plugin_datastore/python_track_store.py | 5 +++++ OTAnalytics/plugin_datastore/track_store.py | 5 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index f8c397722..48e8eea16 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -424,8 +424,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: @abstractmethod def intersection_points( - self, - sections: list[Section], + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: """ Return the intersection points resulting from the tracks and the @@ -439,6 +438,21 @@ def intersection_points( """ raise NotImplementedError + @abstractmethod + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + """Return whether track coordinates are contained by the given sections. + + Args: + sections (Iterable[Section]): the sections. + + Returns: + dict[TrackId, tuple[SectionId, Sequence[bool]]]: boolean mask of track + coordinates contained by given sections. + """ + raise NotImplementedError + @abstractmethod def split(self, chunks: int) -> Sequence["TrackDataset"]: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index dc75e2fa9..3d15f481f 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -318,3 +318,8 @@ def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError + + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index b8e37897d..53a0e2b6a 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -350,6 +350,11 @@ def intersection_points( return intersection_points + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + raise NotImplementedError + def _next_event( self, track_id: str, From d3ae3a8e8b9b164f24412b94dee70a7e7b56f2dd Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:55:18 +0100 Subject: [PATCH 009/107] Implement whether tracks in TrackDataset intersect with given sections --- OTAnalytics/plugin_datastore/track_store.py | 22 ++++++++++++++++++- .../plugin_datastore/test_track_store.py | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 53a0e2b6a..66bc87a1e 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -13,6 +13,7 @@ from pygeos import Geometry, geometrycollections from pygeos import get_coordinates as pygeos_coords from pygeos import intersection as pygeos_intersection +from pygeos import intersects as pygeos_intersects from pygeos import is_empty from pygeos import line_locate_point as pygeos_project from pygeos import linestrings @@ -304,7 +305,26 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": return PandasTrackDataset(filtered_dataset, self._calculator) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - raise NotImplementedError + intersecting_tracks = set() + sections_grouped_by_offset = group_sections_by_offset(sections) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + if (track_df := self._track_geometries.get(offset, None)) is None: + track_df = self._create_track_geom_df(self._dataset, offset) + self._track_geometries[offset] = track_df + + track_df["intersects"] = ( + track_df["geom"] + .apply(lambda line: pygeos_intersects(line, section_geoms)) + .map(any) + .astype(bool) + ) + track_ids = [ + TrackId(_id) for _id in track_df[track_df["intersects"]]["id"].unique() + ] + intersecting_tracks.update(track_ids) + + return intersecting_tracks def intersection_points( self, sections: list[Section] diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index b2f25f630..92365cf14 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -394,3 +394,25 @@ def test_intersection_points( (third_section.id, IntersectionPoint(3)), ], } + + def test_intersecting_tracks( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + dataset = PandasTrackDataset.from_list( + [not_intersecting_track, first_track, second_track] + ) + result = dataset.intersecting_tracks(list(sections)) + assert result == {first_track.id, second_track.id} From 57acfc26dceb6ea74a31fbe9909319fa4f2cdb05 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 10:54:33 +0100 Subject: [PATCH 010/107] Add getting intersection points from PandasTrackDataset with sections --- OTAnalytics/application/analysis/intersect.py | 4 +- OTAnalytics/domain/track.py | 6 +- .../plugin_datastore/python_track_store.py | 4 +- OTAnalytics/plugin_datastore/track_store.py | 114 ++++++++--- .../plugin_datastore/test_track_store.py | 193 +++++++++++++++++- 5 files changed, 278 insertions(+), 43 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index a3bf969f3..61c9b69c7 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from collections import defaultdict -from typing import Iterable, Mapping +from typing import Iterable, Mapping, Sequence from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import RelativeOffsetCoordinate @@ -29,7 +29,7 @@ def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]] def group_sections_by_offset( sections: Iterable[Section], -) -> Mapping[RelativeOffsetCoordinate, Iterable[Section]]: +) -> Mapping[RelativeOffsetCoordinate, Sequence[Section]]: grouped_sections: dict[RelativeOffsetCoordinate, list[Section]] = defaultdict(list) for section in sections: offset = section.get_offset(EventType.SECTION_ENTER) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 31dc15e00..b837bf3d3 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -365,7 +365,9 @@ def __init__(self, track_ids: list[TrackId], message: str): self._track_ids = track_ids -INTERSECTION_COORDINATE = tuple[float, float] +@dataclass +class IntersectionPoint: + index: int class TrackDataset(ABC): @@ -416,7 +418,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section], - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError @abstractmethod diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 0fe1cc7fb..dc75e2fa9 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -9,8 +9,8 @@ from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - INTERSECTION_COORDINATE, Detection, + IntersectionPoint, Track, TrackClassificationCalculator, TrackDataset, @@ -316,5 +316,5 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index bf61645f8..b8e37897d 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -1,26 +1,33 @@ from abc import ABC, abstractmethod +from bisect import bisect +from collections import defaultdict from dataclasses import dataclass from datetime import datetime +from itertools import chain from math import ceil from typing import Any, Iterable, Optional, Sequence import pandas from more_itertools import batched from pandas import DataFrame, Series +from pygeos import Geometry, geometrycollections from pygeos import get_coordinates as pygeos_coords +from pygeos import intersection as pygeos_intersection +from pygeos import is_empty from pygeos import line_locate_point as pygeos_project from pygeos import linestrings from pygeos import points as pygeos_points from pygeos import prepare +from OTAnalytics.application.analysis.intersect import group_sections_by_offset from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - INTERSECTION_COORDINATE, MIN_NUMBER_OF_DETECTIONS, Detection, + IntersectionPoint, Track, TrackDataset, TrackId, @@ -157,11 +164,16 @@ def __init__( self._dataset = dataset self._default_track_offset = default_track_offset self._track_geometries = { - self._default_track_offset: self._create_track_geom_df(self._dataset) + self._default_track_offset: self._create_track_geom_df( + self._dataset, self._default_track_offset + ) } self._calculator = calculator - def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: + @staticmethod + def _create_track_geom_df( + dataset: DataFrame, offset: RelativeOffsetCoordinate + ) -> DataFrame: if dataset.empty: return DataFrame(columns=["id", "geom"]) track_geom_df = DataFrame.from_records( @@ -171,10 +183,8 @@ def _create_track_geom_df(self, dataset: DataFrame) -> DataFrame: linestrings( list( zip( - detections.x - + detections.x * self._default_track_offset.x, - detections.y - + detections.y * self._default_track_offset.y, + detections.x + detections.x * offset.x, + detections.y + detections.y * offset.y, ) ) ), @@ -298,32 +308,62 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: def intersection_points( self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, INTERSECTION_COORDINATE]]]: - # sec_geoms = sections_to_pygeos_multi( - # sections - # ) # [self.section_geom_map[section.id] for section in sections] - # df["intersections"] = df["geom"].apply( - # lambda line: [ - # i for i in pygeos_intersection(line, sec_geoms) if not is_empty(i) - # ] - # ) - # - # vs = ( - # df[df["intersections"].apply(lambda i: len(i) > 0)] - # .apply( - # lambda r: [ - # self.next_event( - # r["track"], r["geom"], pygeos_points(p), r["projection"] - # ) - # for p in pygeos_coords(r["intersections"]) - # ], - # axis=1, - # ) - # .values - # ) - # - # return [v for list in vs for v in list] - raise NotImplementedError + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + sections_grouped_by_offset = group_sections_by_offset(sections) + + intersection_points = defaultdict(list) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + if (track_df := self._track_geometries.get(offset, None)) is None: + track_df = self._create_track_geom_df(self._dataset, offset) + self._track_geometries[offset] = track_df + + track_df["intersections"] = track_df["geom"].apply( + lambda line: [ + (_sections[index].id, ip) + for index, ip in enumerate(pygeos_intersection(line, section_geoms)) + if not is_empty(ip) + ] + ) + intersections = ( + track_df[track_df["intersections"].apply(lambda i: len(i) > 0)] + .apply( + lambda r: [ + self._next_event( + r["id"], + _section_id, + r["geom"], + pygeos_points(p), + r["projection"], + ) + for _section_id, ip in r["intersections"] + for p in pygeos_coords(ip) + ], + axis=1, + ) + .values + ) + for _id, section_id, intersection_point in chain.from_iterable( + intersections + ): + intersection_points[_id].append((section_id, intersection_point)) + + return intersection_points + + def _next_event( + self, + track_id: str, + section_id: SectionId, + track_geom: Geometry, + point: Geometry, + projection: Any, + ) -> tuple[TrackId, SectionId, IntersectionPoint]: + dist = pygeos_project(track_geom, point) + return ( + TrackId(track_id), + section_id, + IntersectionPoint(bisect(projection, dist)), + ) def _assign_track_classification( @@ -372,3 +412,11 @@ def _sort_tracks(track_df: DataFrame) -> DataFrame: return track_df.sort_values([track.TRACK_ID, track.FRAME]) else: return track_df + + +def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: + return geometrycollections([section_to_pygeos(s) for s in sections]) + + +def section_to_pygeos(section: Section) -> Geometry: + return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 1acf08663..b2f25f630 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -2,7 +2,10 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import LineSection, Section, SectionId +from OTAnalytics.domain.track import IntersectionPoint, Track, TrackDataset, TrackId +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasDetection, @@ -16,6 +19,154 @@ ) +@pytest.fixture +def first_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("1") + track_builder.add_xy_bbox(1, 1) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def second_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("2") + track_builder.add_xy_bbox(1, 1.5) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1.5) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1.5) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1.5) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1.5) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("3") + track_builder.add_xy_bbox(1, 10) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 10) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 10) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 10) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 10) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_section() -> Section: + name = "first" + coordinates = [Coordinate(0, 0), Coordinate(0, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def first_section() -> Section: + name = "first" + coordinates = [Coordinate(1.5, 0), Coordinate(1.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def second_section() -> Section: + name = "second" + coordinates = [Coordinate(2.5, 0), Coordinate(2.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def third_section() -> Section: + name = "third" + coordinates = [Coordinate(3.5, 0), Coordinate(3.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + class TestPandasDetection: def test_properties(self) -> None: builder = TrackBuilder() @@ -83,14 +234,15 @@ def test_add(self) -> None: builder.append_detection() builder.append_detection() track = builder.build_track() - detections = builder.build_serialized_detections() - expected_dataset = PandasTrackDataset(DataFrame(detections)) + expected_dataset = PandasTrackDataset.from_list([track]) dataset = PandasTrackDataset() merged = dataset.add_all(PythonTrackDataset({track.id: track})) assert 0 == len(dataset.as_list()) - assert merged == expected_dataset + for actual, expected in zip(merged, expected_dataset): + assert_equal_track_properties(actual, expected) + # assert merged == expected_dataset def test_add_nothing(self) -> None: dataset = PandasTrackDataset() @@ -209,3 +361,36 @@ def test_filter_by_minimum_detection_length(self) -> None: assert len(filtered_dataset) == 1 for actual_track, expected_track in zip(filtered_dataset, [second_track]): assert_equal_track_properties(actual_track, expected_track) + + def test_intersection_points( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + dataset = PandasTrackDataset.from_list( + [not_intersecting_track, first_track, second_track] + ) + result = dataset.intersection_points(list(sections)) + assert result == { + first_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + second_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + } From db50e02aa7d362e46b04aa4af4c6bbfae322de6c Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:03:43 +0100 Subject: [PATCH 011/107] Add documentation --- OTAnalytics/domain/track.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index b837bf3d3..f8c397722 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -412,6 +412,14 @@ def as_list(self) -> list[Track]: @abstractmethod def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ raise NotImplementedError @abstractmethod @@ -419,6 +427,16 @@ def intersection_points( self, sections: list[Section], ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """ + Return the intersection points resulting from the tracks and the + given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId]]]: the intersection points. + """ raise NotImplementedError @abstractmethod From bec63df90c8f91b33e987c7c088c3622510b002c Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:06:33 +0100 Subject: [PATCH 012/107] Extend TrackDataset with method to get boolean mask of track coordinates contained by sections This method will be required when trying to create intersection events with area sections. --- OTAnalytics/domain/track.py | 18 ++++++++++++++++-- .../plugin_datastore/python_track_store.py | 5 +++++ OTAnalytics/plugin_datastore/track_store.py | 5 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index f8c397722..48e8eea16 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -424,8 +424,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: @abstractmethod def intersection_points( - self, - sections: list[Section], + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: """ Return the intersection points resulting from the tracks and the @@ -439,6 +438,21 @@ def intersection_points( """ raise NotImplementedError + @abstractmethod + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + """Return whether track coordinates are contained by the given sections. + + Args: + sections (Iterable[Section]): the sections. + + Returns: + dict[TrackId, tuple[SectionId, Sequence[bool]]]: boolean mask of track + coordinates contained by given sections. + """ + raise NotImplementedError + @abstractmethod def split(self, chunks: int) -> Sequence["TrackDataset"]: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index dc75e2fa9..3d15f481f 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -318,3 +318,8 @@ def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: raise NotImplementedError + + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index b8e37897d..53a0e2b6a 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -350,6 +350,11 @@ def intersection_points( return intersection_points + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + raise NotImplementedError + def _next_event( self, track_id: str, From e3fd49c234d305154ff2b42999cb9c26c3142a26 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:55:18 +0100 Subject: [PATCH 013/107] Implement whether tracks in PandasTrackDataset intersect with given sections --- OTAnalytics/plugin_datastore/track_store.py | 22 ++++++++++++++++++- .../plugin_datastore/test_track_store.py | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 53a0e2b6a..66bc87a1e 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -13,6 +13,7 @@ from pygeos import Geometry, geometrycollections from pygeos import get_coordinates as pygeos_coords from pygeos import intersection as pygeos_intersection +from pygeos import intersects as pygeos_intersects from pygeos import is_empty from pygeos import line_locate_point as pygeos_project from pygeos import linestrings @@ -304,7 +305,26 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": return PandasTrackDataset(filtered_dataset, self._calculator) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - raise NotImplementedError + intersecting_tracks = set() + sections_grouped_by_offset = group_sections_by_offset(sections) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + if (track_df := self._track_geometries.get(offset, None)) is None: + track_df = self._create_track_geom_df(self._dataset, offset) + self._track_geometries[offset] = track_df + + track_df["intersects"] = ( + track_df["geom"] + .apply(lambda line: pygeos_intersects(line, section_geoms)) + .map(any) + .astype(bool) + ) + track_ids = [ + TrackId(_id) for _id in track_df[track_df["intersects"]]["id"].unique() + ] + intersecting_tracks.update(track_ids) + + return intersecting_tracks def intersection_points( self, sections: list[Section] diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index b2f25f630..92365cf14 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -394,3 +394,25 @@ def test_intersection_points( (third_section.id, IntersectionPoint(3)), ], } + + def test_intersecting_tracks( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + dataset = PandasTrackDataset.from_list( + [not_intersecting_track, first_track, second_track] + ) + result = dataset.intersecting_tracks(list(sections)) + assert result == {first_track.id, second_track.id} From 01ca961a64b1645ea04d56c136f49584ac8428c1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:42:07 +0100 Subject: [PATCH 014/107] Refactor track geometry dataset in PandasTrackDataset to own module --- OTAnalytics/domain/track.py | 62 ++++ .../track_geometry_store/pygeos_store.py | 210 ++++++++++++ OTAnalytics/plugin_datastore/track_store.py | 148 +-------- .../plugin_datastore/test_track_store.py | 208 +----------- .../track_geometry_store/test_pygeos_store.py | 312 ++++++++++++++++++ 5 files changed, 595 insertions(+), 345 deletions(-) create mode 100644 OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py create mode 100644 tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 48e8eea16..f30e23002 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -673,3 +673,65 @@ def reset(self) -> None: All configurations made to the builder will be reset. """ raise NotImplementedError + + +class TrackGeometryDataset(ABC): + @staticmethod + @abstractmethod + def from_track_dataset(dataset: TrackDataset) -> "TrackGeometryDataset": + raise NotImplementedError + + @abstractmethod + def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": + raise NotImplementedError + + @abstractmethod + def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": + raise NotImplementedError + + @abstractmethod + def clear(self) -> "TrackGeometryDataset": + raise NotImplementedError + + @abstractmethod + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ + raise NotImplementedError + + @abstractmethod + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """ + Return the intersection points resulting from the tracks and the + given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId]]]: the intersection points. + """ + raise NotImplementedError + + @abstractmethod + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + """Return whether track coordinates are contained by the given sections. + + Args: + sections (Iterable[Section]): the sections. + + Returns: + dict[TrackId, tuple[SectionId, Sequence[bool]]]: boolean mask of track + coordinates contained by given sections. + """ + raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py new file mode 100644 index 000000000..37d833b0d --- /dev/null +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -0,0 +1,210 @@ +from bisect import bisect +from collections import defaultdict +from itertools import chain +from typing import Any, Iterable, Sequence, TypedDict + +from pandas import DataFrame +from pygeos import ( + Geometry, + apply, + geometrycollections, + get_coordinates, + intersection, + intersects, + is_empty, + line_locate_point, + linestrings, + points, + prepare, +) + +from OTAnalytics.application.analysis.intersect import group_sections_by_offset +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset +from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track import ( + IntersectionPoint, + Track, + TrackDataset, + TrackGeometryDataset, + TrackId, +) + + +def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: + return geometrycollections([section_to_pygeos(s) for s in sections]) + + +def section_to_pygeos(section: Section) -> Geometry: + return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) + + +TRACK_ID = "track_id" +GEOMETRY = "geom" +PROJECTION = "projection" +INTERSECTIONS = "intersections" +INTERSECTS = "intersects" +BASE_GEOMETRY = RelativeOffsetCoordinate(0, 0) + + +class TrackGeometryEntry(TypedDict): + geometry: Geometry + projection: list[float] + intersection: list + intersects: list + + +class PygeosTrackGeometryDataset(TrackGeometryDataset): + def __init__( + self, + dataset: dict[RelativeOffsetCoordinate, DataFrame], + ): + self._dataset: dict[RelativeOffsetCoordinate, DataFrame] = dataset + + @staticmethod + def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: + if len(dataset) == 0: + return PygeosTrackGeometryDataset( + {BASE_GEOMETRY: DataFrame(columns=[TRACK_ID, GEOMETRY])} + ) + + tracks = [PygeosTrackGeometryDataset._create_track(track) for track in dataset] + # Question: Is DataFrame.from_dict faster than DataFrame.from_records + track_geom_df = DataFrame.from_records(tracks, columns=[TRACK_ID, GEOMETRY]) + track_geom_df[PROJECTION] = track_geom_df[GEOMETRY].apply( + lambda track_geom: [ + line_locate_point(track_geom, points(p)) + for p in get_coordinates(track_geom) + ] + ) + track_geom_df[GEOMETRY].apply(prepare) + + return PygeosTrackGeometryDataset({BASE_GEOMETRY: track_geom_df}) + + @staticmethod + def _create_track( + track: Track, offset: RelativeOffsetCoordinate | None = None + ) -> tuple[str, Geometry]: + if offset: + return track.id.id, linestrings( + [ + apply_offset( + detection.x, detection.y, detection.w, detection.h, offset + ) + for detection in track.detections + ] + ) + return track.id.id, linestrings( + [(detection.x, detection.y) for detection in track.detections] + ) + + def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: + raise NotImplementedError + + def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: + raise NotImplementedError + + def clear(self) -> TrackGeometryDataset: + raise NotImplementedError + + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + intersecting_tracks = set() + sections_grouped_by_offset = group_sections_by_offset(sections) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + track_df = self._get_track_geometries_for(offset) + + track_df[INTERSECTS] = ( + track_df[GEOMETRY] + .apply(lambda line: intersects(line, section_geoms)) + .map(any) + .astype(bool) + ) + track_ids = [ + TrackId(_id) + for _id in track_df[track_df[INTERSECTS]][TRACK_ID].unique() + ] + intersecting_tracks.update(track_ids) + + return intersecting_tracks + + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + sections_grouped_by_offset = group_sections_by_offset(sections) + + intersection_points = defaultdict(list) + for offset, _sections in sections_grouped_by_offset.items(): + section_geoms = sections_to_pygeos_multi(_sections) + track_df = self._get_track_geometries_for(offset) + + track_df[INTERSECTIONS] = track_df[GEOMETRY].apply( + lambda line: [ + (_sections[index].id, ip) + for index, ip in enumerate(intersection(line, section_geoms)) + if not is_empty(ip) + ] + ) + intersections = ( + track_df[track_df[INTERSECTIONS].apply(lambda i: len(i) > 0)] + .apply( + lambda r: [ + self._next_event( + r[TRACK_ID], + _section_id, + r[GEOMETRY], + points(p), + r[PROJECTION], + ) + for _section_id, ip in r[INTERSECTIONS] + for p in get_coordinates(ip) + ], + axis=1, + ) + .values + ) + for _id, section_id, intersection_point in chain.from_iterable( + intersections + ): + intersection_points[_id].append((section_id, intersection_point)) + + return intersection_points + + def _get_track_geometries_for(self, offset: RelativeOffsetCoordinate) -> DataFrame: + if (track_df := self._dataset.get(offset, None)) is None: + self._create_tracks_for(offset) + track_df = self._dataset[offset] + return track_df + + def _create_tracks_for(self, offset: RelativeOffsetCoordinate) -> None: + base_track_geometry = self._dataset[BASE_GEOMETRY] + self._dataset[offset] = self._apply_offset(base_track_geometry, offset) + + @staticmethod + def _apply_offset( + base_track_geometries: DataFrame, offset: RelativeOffsetCoordinate + ) -> DataFrame: + new_track_df = base_track_geometries.copy() + new_track_df[GEOMETRY] = new_track_df[GEOMETRY].apply( + lambda geom: apply(geom, lambda coord: coord + coord * [offset.x, offset.y]) + ) + return new_track_df + + def _next_event( + self, + track_id: str, + section_id: SectionId, + track_geom: Geometry, + point: Geometry, + projection: Any, + ) -> tuple[TrackId, SectionId, IntersectionPoint]: + dist = line_locate_point(track_geom, point) + return ( + TrackId(track_id), + section_id, + IntersectionPoint(bisect(projection, dist)), + ) + + def contained_by_sections( + self, sections: Iterable[Section] + ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 66bc87a1e..06b30b382 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -1,29 +1,14 @@ from abc import ABC, abstractmethod -from bisect import bisect -from collections import defaultdict from dataclasses import dataclass from datetime import datetime -from itertools import chain from math import ceil from typing import Any, Iterable, Optional, Sequence import pandas from more_itertools import batched from pandas import DataFrame, Series -from pygeos import Geometry, geometrycollections -from pygeos import get_coordinates as pygeos_coords -from pygeos import intersection as pygeos_intersection -from pygeos import intersects as pygeos_intersects -from pygeos import is_empty -from pygeos import line_locate_point as pygeos_project -from pygeos import linestrings -from pygeos import points as pygeos_points -from pygeos import prepare - -from OTAnalytics.application.analysis.intersect import group_sections_by_offset -from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET + from OTAnalytics.domain import track -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( MIN_NUMBER_OF_DETECTIONS, @@ -33,6 +18,10 @@ TrackDataset, TrackId, ) +from OTAnalytics.plugin_datastore.track_geometry_store import ( + TRACK_GEOMETRY_DATASET_FACTORY, + TRACK_GEOMETRY_FACTORY, +) class PandasDetection(Detection): @@ -160,48 +149,12 @@ def __init__( self, dataset: DataFrame = DataFrame(), calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, - default_track_offset: RelativeOffsetCoordinate = DEFAULT_TRACK_OFFSET, + track_geometry_factory: TRACK_GEOMETRY_FACTORY = TRACK_GEOMETRY_DATASET_FACTORY, ): self._dataset = dataset - self._default_track_offset = default_track_offset - self._track_geometries = { - self._default_track_offset: self._create_track_geom_df( - self._dataset, self._default_track_offset - ) - } self._calculator = calculator - - @staticmethod - def _create_track_geom_df( - dataset: DataFrame, offset: RelativeOffsetCoordinate - ) -> DataFrame: - if dataset.empty: - return DataFrame(columns=["id", "geom"]) - track_geom_df = DataFrame.from_records( - [ - ( - track_id, - linestrings( - list( - zip( - detections.x + detections.x * offset.x, - detections.y + detections.y * offset.y, - ) - ) - ), - ) - for track_id, detections in dataset.groupby(track.TRACK_ID) - ], - columns=["id", "geom"], - ) - track_geom_df["projection"] = track_geom_df["geom"].apply( - lambda track_geom: [ - pygeos_project(track_geom, pygeos_points(p)) - for p in pygeos_coords(track_geom) - ] - ) - track_geom_df["geom"].apply(prepare) - return track_geom_df + self._track_geometry_factory = track_geometry_factory + self._track_geometry_dataset = self._track_geometry_factory(self) @staticmethod def from_list( @@ -305,91 +258,18 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": return PandasTrackDataset(filtered_dataset, self._calculator) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - intersecting_tracks = set() - sections_grouped_by_offset = group_sections_by_offset(sections) - for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = sections_to_pygeos_multi(_sections) - if (track_df := self._track_geometries.get(offset, None)) is None: - track_df = self._create_track_geom_df(self._dataset, offset) - self._track_geometries[offset] = track_df - - track_df["intersects"] = ( - track_df["geom"] - .apply(lambda line: pygeos_intersects(line, section_geoms)) - .map(any) - .astype(bool) - ) - track_ids = [ - TrackId(_id) for _id in track_df[track_df["intersects"]]["id"].unique() - ] - intersecting_tracks.update(track_ids) - - return intersecting_tracks + return self._track_geometry_dataset.intersecting_tracks(sections) def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - sections_grouped_by_offset = group_sections_by_offset(sections) - - intersection_points = defaultdict(list) - for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = sections_to_pygeos_multi(_sections) - if (track_df := self._track_geometries.get(offset, None)) is None: - track_df = self._create_track_geom_df(self._dataset, offset) - self._track_geometries[offset] = track_df - - track_df["intersections"] = track_df["geom"].apply( - lambda line: [ - (_sections[index].id, ip) - for index, ip in enumerate(pygeos_intersection(line, section_geoms)) - if not is_empty(ip) - ] - ) - intersections = ( - track_df[track_df["intersections"].apply(lambda i: len(i) > 0)] - .apply( - lambda r: [ - self._next_event( - r["id"], - _section_id, - r["geom"], - pygeos_points(p), - r["projection"], - ) - for _section_id, ip in r["intersections"] - for p in pygeos_coords(ip) - ], - axis=1, - ) - .values - ) - for _id, section_id, intersection_point in chain.from_iterable( - intersections - ): - intersection_points[_id].append((section_id, intersection_point)) - - return intersection_points + return self._track_geometry_dataset.intersection_points(sections) def contained_by_sections( self, sections: Iterable[Section] ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: raise NotImplementedError - def _next_event( - self, - track_id: str, - section_id: SectionId, - track_geom: Geometry, - point: Geometry, - projection: Any, - ) -> tuple[TrackId, SectionId, IntersectionPoint]: - dist = pygeos_project(track_geom, point) - return ( - TrackId(track_id), - section_id, - IntersectionPoint(bisect(projection, dist)), - ) - def _assign_track_classification( data: DataFrame, calculator: PandasTrackClassificationCalculator @@ -437,11 +317,3 @@ def _sort_tracks(track_df: DataFrame) -> DataFrame: return track_df.sort_values([track.TRACK_ID, track.FRAME]) else: return track_df - - -def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: - return geometrycollections([section_to_pygeos(s) for s in sections]) - - -def section_to_pygeos(section: Section) -> Geometry: - return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 92365cf14..4c34d4d70 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -2,10 +2,7 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.section import LineSection, Section, SectionId -from OTAnalytics.domain.track import IntersectionPoint, Track, TrackDataset, TrackId -from OTAnalytics.domain.types import EventType +from OTAnalytics.domain.track import Track, TrackDataset, TrackId from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasDetection, @@ -19,154 +16,6 @@ ) -@pytest.fixture -def first_track() -> Track: - track_builder = TrackBuilder() - track_builder.add_track_id("1") - track_builder.add_xy_bbox(1, 1) - track_builder.add_frame(1) - track_builder.add_second(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(2, 1) - track_builder.add_frame(2) - track_builder.add_second(2) - track_builder.append_detection() - - track_builder.add_xy_bbox(3, 1) - track_builder.add_frame(3) - track_builder.add_second(3) - track_builder.append_detection() - - track_builder.add_xy_bbox(4, 1) - track_builder.add_frame(4) - track_builder.add_second(4) - track_builder.append_detection() - - track_builder.add_xy_bbox(5, 1) - track_builder.add_frame(5) - track_builder.add_second(5) - track_builder.append_detection() - - return track_builder.build_track() - - -@pytest.fixture -def second_track() -> Track: - track_builder = TrackBuilder() - track_builder.add_track_id("2") - track_builder.add_xy_bbox(1, 1.5) - track_builder.add_frame(1) - track_builder.add_second(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(2, 1.5) - track_builder.add_frame(2) - track_builder.add_second(2) - track_builder.append_detection() - - track_builder.add_xy_bbox(3, 1.5) - track_builder.add_frame(3) - track_builder.add_second(3) - track_builder.append_detection() - - track_builder.add_xy_bbox(4, 1.5) - track_builder.add_frame(4) - track_builder.add_second(4) - track_builder.append_detection() - - track_builder.add_xy_bbox(5, 1.5) - track_builder.add_frame(5) - track_builder.add_second(5) - track_builder.append_detection() - - return track_builder.build_track() - - -@pytest.fixture -def not_intersecting_track() -> Track: - track_builder = TrackBuilder() - track_builder.add_track_id("3") - track_builder.add_xy_bbox(1, 10) - track_builder.add_frame(1) - track_builder.add_second(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(2, 10) - track_builder.add_frame(2) - track_builder.add_second(2) - track_builder.append_detection() - - track_builder.add_xy_bbox(3, 10) - track_builder.add_frame(3) - track_builder.add_second(3) - track_builder.append_detection() - - track_builder.add_xy_bbox(4, 10) - track_builder.add_frame(4) - track_builder.add_second(4) - track_builder.append_detection() - - track_builder.add_xy_bbox(5, 10) - track_builder.add_frame(5) - track_builder.add_second(5) - track_builder.append_detection() - - return track_builder.build_track() - - -@pytest.fixture -def not_intersecting_section() -> Section: - name = "first" - coordinates = [Coordinate(0, 0), Coordinate(0, 2)] - return LineSection( - SectionId(name), - name, - {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, - {}, - coordinates, - ) - - -@pytest.fixture -def first_section() -> Section: - name = "first" - coordinates = [Coordinate(1.5, 0), Coordinate(1.5, 2)] - return LineSection( - SectionId(name), - name, - {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, - {}, - coordinates, - ) - - -@pytest.fixture -def second_section() -> Section: - name = "second" - coordinates = [Coordinate(2.5, 0), Coordinate(2.5, 2)] - return LineSection( - SectionId(name), - name, - {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, - {}, - coordinates, - ) - - -@pytest.fixture -def third_section() -> Section: - name = "third" - coordinates = [Coordinate(3.5, 0), Coordinate(3.5, 2)] - return LineSection( - SectionId(name), - name, - {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, - {}, - coordinates, - ) - - class TestPandasDetection: def test_properties(self) -> None: builder = TrackBuilder() @@ -361,58 +210,3 @@ def test_filter_by_minimum_detection_length(self) -> None: assert len(filtered_dataset) == 1 for actual_track, expected_track in zip(filtered_dataset, [second_track]): assert_equal_track_properties(actual_track, expected_track) - - def test_intersection_points( - self, - not_intersecting_track: Track, - first_track: Track, - second_track: Track, - not_intersecting_section: Section, - first_section: Section, - second_section: Section, - third_section: Section, - ) -> None: - sections = [ - not_intersecting_section, - first_section, - second_section, - third_section, - ] - dataset = PandasTrackDataset.from_list( - [not_intersecting_track, first_track, second_track] - ) - result = dataset.intersection_points(list(sections)) - assert result == { - first_track.id: [ - (first_section.id, IntersectionPoint(1)), - (second_section.id, IntersectionPoint(2)), - (third_section.id, IntersectionPoint(3)), - ], - second_track.id: [ - (first_section.id, IntersectionPoint(1)), - (second_section.id, IntersectionPoint(2)), - (third_section.id, IntersectionPoint(3)), - ], - } - - def test_intersecting_tracks( - self, - not_intersecting_track: Track, - first_track: Track, - second_track: Track, - not_intersecting_section: Section, - first_section: Section, - second_section: Section, - third_section: Section, - ) -> None: - sections = [ - not_intersecting_section, - first_section, - second_section, - third_section, - ] - dataset = PandasTrackDataset.from_list( - [not_intersecting_track, first_track, second_track] - ) - result = dataset.intersecting_tracks(list(sections)) - assert result == {first_track.id, second_track.id} diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py new file mode 100644 index 000000000..fc2326a57 --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -0,0 +1,312 @@ +from unittest.mock import MagicMock, Mock + +import pytest +from pandas import DataFrame +from pygeos import Geometry, linestrings + +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import LineSection, Section, SectionId +from OTAnalytics.domain.track import ( + IntersectionPoint, + Track, + TrackDataset, + TrackGeometryDataset, + TrackId, +) +from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + BASE_GEOMETRY, + GEOMETRY, + PROJECTION, + TRACK_ID, + PygeosTrackGeometryDataset, +) +from tests.conftest import TrackBuilder + + +def create_pygeos_track(track: Track) -> Geometry: + return linestrings([(detection.x, detection.y) for detection in track.detections]) + + +def create_track_dataset(tracks: list[Track]) -> TrackDataset: + dataset = MagicMock() + dataset.__iter__.return_value = iter(tracks) + dataset.__len__.return_value = len(tracks) + return dataset + + +@pytest.fixture +def first_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("1") + track_builder.add_xy_bbox(1, 1) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def second_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("2") + track_builder.add_xy_bbox(1, 1.5) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 1.5) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 1.5) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 1.5) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 1.5) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("3") + track_builder.add_xy_bbox(1, 10) + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2, 10) + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(3, 10) + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(4, 10) + track_builder.add_frame(4) + track_builder.add_second(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(5, 10) + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def not_intersecting_section() -> Section: + name = "first" + coordinates = [Coordinate(0, 0), Coordinate(0, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def first_section() -> Section: + name = "first" + coordinates = [Coordinate(1.5, 0), Coordinate(1.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def second_section() -> Section: + name = "second" + coordinates = [Coordinate(2.5, 0), Coordinate(2.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def third_section() -> Section: + name = "third" + coordinates = [Coordinate(3.5, 0), Coordinate(3.5, 2)] + return LineSection( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +def assert_track_geometry_dataset_equals( + to_compare: TrackGeometryDataset, other: TrackGeometryDataset +) -> None: + assert isinstance(to_compare, PygeosTrackGeometryDataset) + assert isinstance(other, PygeosTrackGeometryDataset) + assert to_compare._dataset.keys() == other._dataset.keys() # noqa + + for offset, track_geom in to_compare._dataset.items(): # noqa + assert track_geom.equals(other._dataset[offset]) # noqa + + +class TestPygeosTrackGeometryDataset: + @pytest.fixture + def simple_track(self) -> Track: + first_detection = Mock() + first_detection.x = 1 + first_detection.y = 0 + second_detection = Mock() + second_detection.x = 2 + second_detection.y = 0 + simple_track = Mock() + simple_track.id = TrackId("1") + simple_track.detections = [first_detection, second_detection] + return simple_track + + def test_from_track_dataset(self, simple_track: Track) -> None: + track_dataset = create_track_dataset([simple_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) + expected = PygeosTrackGeometryDataset( + { + BASE_GEOMETRY: DataFrame.from_records( + [ + ( + simple_track.id.id, + create_pygeos_track(simple_track), + [0.0, 1.0], + ), + ], + columns=[TRACK_ID, GEOMETRY, PROJECTION], + ) + } + ) + assert_track_geometry_dataset_equals(geometry_dataset, expected) + + @pytest.mark.skip + def test_add_all(self, first_track: Track, second_track: Track) -> None: + track_dataset = create_track_dataset([first_track, second_track]) + geometry_dataset = PygeosTrackGeometryDataset({}) + geometry_dataset.add_all(track_dataset) + + expected = PygeosTrackGeometryDataset( + { + BASE_GEOMETRY: DataFrame.from_records( + [ + ( + first_track.id.id, + create_pygeos_track(first_track), + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + ), + ( + second_track.id.id, + create_pygeos_track(second_track), + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + ), + ] + ) + } + ) + assert_track_geometry_dataset_equals(geometry_dataset, expected) + + def test_intersection_points( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + track_dataset = create_track_dataset( + [not_intersecting_track, first_track, second_track] + ) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.intersection_points(list(sections)) + assert result == { + first_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + second_track.id: [ + (first_section.id, IntersectionPoint(1)), + (second_section.id, IntersectionPoint(2)), + (third_section.id, IntersectionPoint(3)), + ], + } + + def test_intersecting_tracks( + self, + not_intersecting_track: Track, + first_track: Track, + second_track: Track, + not_intersecting_section: Section, + first_section: Section, + second_section: Section, + third_section: Section, + ) -> None: + sections = [ + not_intersecting_section, + first_section, + second_section, + third_section, + ] + track_dataset = create_track_dataset( + [not_intersecting_track, first_track, second_track] + ) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.intersecting_tracks(list(sections)) + assert result == {first_track.id, second_track.id} From 90e221dc46f03685f17cca8c637bd24ac56662d9 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:43:57 +0100 Subject: [PATCH 015/107] Move constants to top --- .../track_geometry_store/pygeos_store.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 37d833b0d..233eea07f 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -29,6 +29,13 @@ TrackId, ) +TRACK_ID = "track_id" +GEOMETRY = "geom" +PROJECTION = "projection" +INTERSECTIONS = "intersections" +INTERSECTS = "intersects" +BASE_GEOMETRY = RelativeOffsetCoordinate(0, 0) + def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: return geometrycollections([section_to_pygeos(s) for s in sections]) @@ -38,14 +45,6 @@ def section_to_pygeos(section: Section) -> Geometry: return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) -TRACK_ID = "track_id" -GEOMETRY = "geom" -PROJECTION = "projection" -INTERSECTIONS = "intersections" -INTERSECTS = "intersects" -BASE_GEOMETRY = RelativeOffsetCoordinate(0, 0) - - class TrackGeometryEntry(TypedDict): geometry: Geometry projection: list[float] From f32604dd5568d619853ad7c71f10dcbd21e8ac83 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:11:58 +0100 Subject: [PATCH 016/107] Add method to serialise PygeosTrackGeometryDataset to dictionary --- .../track_geometry_store/pygeos_store.py | 10 ++- .../track_geometry_store/test_pygeos_store.py | 71 ++++++++++--------- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 233eea07f..740c47c61 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -1,7 +1,7 @@ from bisect import bisect from collections import defaultdict from itertools import chain -from typing import Any, Iterable, Sequence, TypedDict +from typing import Any, Iterable, Literal, Sequence, TypedDict from pandas import DataFrame from pygeos import ( @@ -35,6 +35,7 @@ INTERSECTIONS = "intersections" INTERSECTS = "intersects" BASE_GEOMETRY = RelativeOffsetCoordinate(0, 0) +ORIENTATION_INDEX: Literal["index"] = "index" def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: @@ -207,3 +208,10 @@ def contained_by_sections( self, sections: Iterable[Section] ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: raise NotImplementedError + + def as_dict(self) -> dict: + result = {} + for offset, track_geom_df in self._dataset.items(): + result[offset] = track_geom_df.to_dict(orient=ORIENTATION_INDEX) + + return result diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index fc2326a57..0359df0c5 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -1,8 +1,9 @@ +from typing import Iterable from unittest.mock import MagicMock, Mock import pytest from pandas import DataFrame -from pygeos import Geometry, linestrings +from pygeos import Geometry, get_coordinates, line_locate_point, linestrings, points from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import LineSection, Section, SectionId @@ -35,6 +36,25 @@ def create_track_dataset(tracks: list[Track]) -> TrackDataset: return dataset +def create_geometry_dataset_from(tracks: Iterable[Track]) -> PygeosTrackGeometryDataset: + entries = [] + for track in tracks: + _id = track.id.id + geometry = create_pygeos_track(track) + projection = [ + line_locate_point(geometry, points(p)) for p in get_coordinates(geometry) + ] + entries.append((_id, geometry, projection)) + return PygeosTrackGeometryDataset( + { + BASE_GEOMETRY: DataFrame.from_records( + entries, + columns=[TRACK_ID, GEOMETRY, PROJECTION], + ) + } + ) + + @pytest.fixture def first_track() -> Track: track_builder = TrackBuilder() @@ -212,20 +232,8 @@ def test_from_track_dataset(self, simple_track: Track) -> None: track_dataset = create_track_dataset([simple_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) - expected = PygeosTrackGeometryDataset( - { - BASE_GEOMETRY: DataFrame.from_records( - [ - ( - simple_track.id.id, - create_pygeos_track(simple_track), - [0.0, 1.0], - ), - ], - columns=[TRACK_ID, GEOMETRY, PROJECTION], - ) - } - ) + + expected = create_geometry_dataset_from([simple_track]) assert_track_geometry_dataset_equals(geometry_dataset, expected) @pytest.mark.skip @@ -233,25 +241,7 @@ def test_add_all(self, first_track: Track, second_track: Track) -> None: track_dataset = create_track_dataset([first_track, second_track]) geometry_dataset = PygeosTrackGeometryDataset({}) geometry_dataset.add_all(track_dataset) - - expected = PygeosTrackGeometryDataset( - { - BASE_GEOMETRY: DataFrame.from_records( - [ - ( - first_track.id.id, - create_pygeos_track(first_track), - [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], - ), - ( - second_track.id.id, - create_pygeos_track(second_track), - [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], - ), - ] - ) - } - ) + expected = create_geometry_dataset_from([first_track, second_track]) assert_track_geometry_dataset_equals(geometry_dataset, expected) def test_intersection_points( @@ -310,3 +300,16 @@ def test_intersecting_tracks( geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) result = geometry_dataset.intersecting_tracks(list(sections)) assert result == {first_track.id, second_track.id} + + def test_as_dict(self, first_track: Track, second_track: Track) -> None: + tracks = [first_track, second_track] + track_dataset = create_track_dataset(tracks) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) + result = geometry_dataset.as_dict() + expected = { + BASE_GEOMETRY: create_geometry_dataset_from(tracks) + ._dataset[BASE_GEOMETRY] + .to_dict(orient="index") + } + assert result == expected From 53ac4a0b10527c7e0221e4616831207e09015f8e Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:51:07 +0100 Subject: [PATCH 017/107] Change way to create PygeosTrackGeometryDataset The track id column is now used as the DataFrame index. Thus, there is no column for track ids anymore. --- .../track_geometry_store/pygeos_store.py | 68 ++++++++++++++----- .../track_geometry_store/test_pygeos_store.py | 2 +- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 740c47c61..3fdb28804 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -34,6 +34,7 @@ PROJECTION = "projection" INTERSECTIONS = "intersections" INTERSECTS = "intersects" +COLUMNS = [GEOMETRY, PROJECTION] BASE_GEOMETRY = RelativeOffsetCoordinate(0, 0) ORIENTATION_INDEX: Literal["index"] = "index" @@ -64,28 +65,58 @@ def __init__( def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: if len(dataset) == 0: return PygeosTrackGeometryDataset( - {BASE_GEOMETRY: DataFrame(columns=[TRACK_ID, GEOMETRY])} + {BASE_GEOMETRY: DataFrame(columns=COLUMNS)} ) - - tracks = [PygeosTrackGeometryDataset._create_track(track) for track in dataset] - # Question: Is DataFrame.from_dict faster than DataFrame.from_records - track_geom_df = DataFrame.from_records(tracks, columns=[TRACK_ID, GEOMETRY]) - track_geom_df[PROJECTION] = track_geom_df[GEOMETRY].apply( - lambda track_geom: [ - line_locate_point(track_geom, points(p)) - for p in get_coordinates(track_geom) - ] + track_geom_df = DataFrame.from_dict( + PygeosTrackGeometryDataset._create_entries(dataset), + columns=COLUMNS, + orient=ORIENTATION_INDEX, ) - track_geom_df[GEOMETRY].apply(prepare) - return PygeosTrackGeometryDataset({BASE_GEOMETRY: track_geom_df}) + @staticmethod + def _create_entries(tracks: Iterable[Track]) -> dict: + """Create track geometry entries from given tracks. + + The resulting dictionary has following the structure: + {TRACK_ID: {GEOMETRY: Geometry, PROJECTION: list[float]}} + + Args: + tracks (Iterable[Track]): the tracks to create the entries from. + + Returns: + dict: the entries. + """ + entries = dict() + for track in tracks: + track_id = track.id.id + geometry = PygeosTrackGeometryDataset._create_track(track) + projection = [ + line_locate_point(geometry, points(p)) + for p in get_coordinates(geometry) + ] + entries[track_id] = { + GEOMETRY: geometry, + PROJECTION: projection, + } + return entries + @staticmethod def _create_track( track: Track, offset: RelativeOffsetCoordinate | None = None - ) -> tuple[str, Geometry]: + ) -> Geometry: + """Creates a prepared pygeos LINESTRING for given track. + + Args: + track (Track): the track. + offset (RelativeOffsetCoordinate | None): the offset to be applied to + geometry. Defaults to None. + + Returns: + Geometry: the prepared pygeos geometry. + """ if offset: - return track.id.id, linestrings( + geometry = linestrings( [ apply_offset( detection.x, detection.y, detection.w, detection.h, offset @@ -93,9 +124,12 @@ def _create_track( for detection in track.detections ] ) - return track.id.id, linestrings( - [(detection.x, detection.y) for detection in track.detections] - ) + else: + geometry = linestrings( + [(detection.x, detection.y) for detection in track.detections] + ) + prepare(geometry) + return geometry def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: raise NotImplementedError diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 0359df0c5..2272b3e6a 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -50,7 +50,7 @@ def create_geometry_dataset_from(tracks: Iterable[Track]) -> PygeosTrackGeometry BASE_GEOMETRY: DataFrame.from_records( entries, columns=[TRACK_ID, GEOMETRY, PROJECTION], - ) + ).set_index(TRACK_ID) } ) From dc42018ec0640640083866dac787877a8e4e62f5 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 16:15:55 +0100 Subject: [PATCH 018/107] Fix access to get track ids from track geometry DataFrame Use DataFrame index to access track ids. --- .../plugin_datastore/track_geometry_store/pygeos_store.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 3fdb28804..f65998808 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -153,10 +153,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: .map(any) .astype(bool) ) - track_ids = [ - TrackId(_id) - for _id in track_df[track_df[INTERSECTS]][TRACK_ID].unique() - ] + track_ids = [TrackId(_id) for _id in track_df[track_df[INTERSECTS]].index] intersecting_tracks.update(track_ids) return intersecting_tracks @@ -183,7 +180,7 @@ def intersection_points( .apply( lambda r: [ self._next_event( - r[TRACK_ID], + r.name, # the track id (track ids is used as df index) _section_id, r[GEOMETRY], points(p), From 65786422a1f77cc4613c1b8cc25f961925ca9c24 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 16:18:08 +0100 Subject: [PATCH 019/107] Extract method to create pygeos track objects from PyGeosTrackDataset as factory method --- .../track_geometry_store/pygeos_store.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index f65998808..fc81eaef7 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -47,6 +47,34 @@ def section_to_pygeos(section: Section) -> Geometry: return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) +def create_pygeos_track( + track: Track, offset: RelativeOffsetCoordinate | None = None +) -> Geometry: + """Creates a prepared pygeos LINESTRING for given track. + + Args: + track (Track): the track. + offset (RelativeOffsetCoordinate | None): the offset to be applied to + geometry. Defaults to None. + + Returns: + Geometry: the prepared pygeos geometry. + """ + if offset: + geometry = linestrings( + [ + apply_offset(detection.x, detection.y, detection.w, detection.h, offset) + for detection in track.detections + ] + ) + else: + geometry = linestrings( + [(detection.x, detection.y) for detection in track.detections] + ) + prepare(geometry) + return geometry + + class TrackGeometryEntry(TypedDict): geometry: Geometry projection: list[float] @@ -90,7 +118,7 @@ def _create_entries(tracks: Iterable[Track]) -> dict: entries = dict() for track in tracks: track_id = track.id.id - geometry = PygeosTrackGeometryDataset._create_track(track) + geometry = create_pygeos_track(track) projection = [ line_locate_point(geometry, points(p)) for p in get_coordinates(geometry) From 7e9081f45b25b1ee52fd6543c32202580103a775 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 16:43:43 +0100 Subject: [PATCH 020/107] Disregard intersections and intersects column when getting dictionary representation of PygeosTrackGeometryDataset --- .../plugin_datastore/track_geometry_store/pygeos_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index fc81eaef7..0495a8b9d 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -271,6 +271,6 @@ def contained_by_sections( def as_dict(self) -> dict: result = {} for offset, track_geom_df in self._dataset.items(): - result[offset] = track_geom_df.to_dict(orient=ORIENTATION_INDEX) + result[offset] = track_geom_df[COLUMNS].to_dict(orient=ORIENTATION_INDEX) return result From 5ab8275b8150b2ce391fb7e1bbe32c5cd9da8122 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:26:30 +0100 Subject: [PATCH 021/107] Ensure correct instantiation of PyGeosTrackGeometryDataset --- .../track_geometry_store/pygeos_store.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 0495a8b9d..285ce6b9e 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -82,19 +82,36 @@ class TrackGeometryEntry(TypedDict): intersects: list +class InvalidTrackGeometryDataset(Exception): + pass + + class PygeosTrackGeometryDataset(TrackGeometryDataset): def __init__( self, - dataset: dict[RelativeOffsetCoordinate, DataFrame], + dataset: dict[RelativeOffsetCoordinate, DataFrame] | None = None, ): - self._dataset: dict[RelativeOffsetCoordinate, DataFrame] = dataset + if dataset is not None: + self._check_is_valid(dataset) + self._dataset: dict[RelativeOffsetCoordinate, DataFrame] = dataset + else: + self._dataset = self._create_empty() + + def _create_empty(self) -> dict[RelativeOffsetCoordinate, DataFrame]: + return {BASE_GEOMETRY: DataFrame(columns=COLUMNS)} + + def _check_is_valid( + self, dataset: dict[RelativeOffsetCoordinate, DataFrame] + ) -> None: + try: + dataset[BASE_GEOMETRY] + except KeyError: + raise InvalidTrackGeometryDataset(f"Missing entry for key {BASE_GEOMETRY}") @staticmethod def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: if len(dataset) == 0: - return PygeosTrackGeometryDataset( - {BASE_GEOMETRY: DataFrame(columns=COLUMNS)} - ) + return PygeosTrackGeometryDataset() track_geom_df = DataFrame.from_dict( PygeosTrackGeometryDataset._create_entries(dataset), columns=COLUMNS, From a0711603c1584479d9b76b33875f4f89db1e5763 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:30:26 +0100 Subject: [PATCH 022/107] Use BASE_GEOMETRY as default offset when creating new track geometry entries --- .../track_geometry_store/pygeos_store.py | 24 +++++++++++-------- .../track_geometry_store/test_pygeos_store.py | 3 ++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 285ce6b9e..9d8421f0a 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -48,29 +48,29 @@ def section_to_pygeos(section: Section) -> Geometry: def create_pygeos_track( - track: Track, offset: RelativeOffsetCoordinate | None = None + track: Track, offset: RelativeOffsetCoordinate = BASE_GEOMETRY ) -> Geometry: """Creates a prepared pygeos LINESTRING for given track. Args: track (Track): the track. - offset (RelativeOffsetCoordinate | None): the offset to be applied to - geometry. Defaults to None. + offset (RelativeOffsetCoordinate): the offset to be applied to + geometry. Returns: Geometry: the prepared pygeos geometry. """ - if offset: + if offset == BASE_GEOMETRY: + geometry = linestrings( + [(detection.x, detection.y) for detection in track.detections] + ) + else: geometry = linestrings( [ apply_offset(detection.x, detection.y, detection.w, detection.h, offset) for detection in track.detections ] ) - else: - geometry = linestrings( - [(detection.x, detection.y) for detection in track.detections] - ) prepare(geometry) return geometry @@ -120,7 +120,9 @@ def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: return PygeosTrackGeometryDataset({BASE_GEOMETRY: track_geom_df}) @staticmethod - def _create_entries(tracks: Iterable[Track]) -> dict: + def _create_entries( + tracks: Iterable[Track], offset: RelativeOffsetCoordinate = BASE_GEOMETRY + ) -> dict: """Create track geometry entries from given tracks. The resulting dictionary has following the structure: @@ -128,6 +130,8 @@ def _create_entries(tracks: Iterable[Track]) -> dict: Args: tracks (Iterable[Track]): the tracks to create the entries from. + offset (RelativeOffsetCoordinate): the offset to apply to the tracks. + Defaults to BASE_GEOMETRY. Returns: dict: the entries. @@ -135,7 +139,7 @@ def _create_entries(tracks: Iterable[Track]) -> dict: entries = dict() for track in tracks: track_id = track.id.id - geometry = create_pygeos_track(track) + geometry = create_pygeos_track(track, offset) projection = [ line_locate_point(geometry, points(p)) for p in get_coordinates(geometry) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 2272b3e6a..a100eff49 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -17,6 +17,7 @@ from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( BASE_GEOMETRY, + COLUMNS, GEOMETRY, PROJECTION, TRACK_ID, @@ -309,7 +310,7 @@ def test_as_dict(self, first_track: Track, second_track: Track) -> None: result = geometry_dataset.as_dict() expected = { BASE_GEOMETRY: create_geometry_dataset_from(tracks) - ._dataset[BASE_GEOMETRY] + ._dataset[BASE_GEOMETRY][COLUMNS] .to_dict(orient="index") } assert result == expected From c5ffe1221e1e63c474e0f9de522b90305a1f11f7 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:41:40 +0100 Subject: [PATCH 023/107] Add property informing whether PygeosTrackGeometryDataset is empty --- .../track_geometry_store/pygeos_store.py | 7 +++++++ .../track_geometry_store/test_pygeos_store.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 9d8421f0a..17922a8c0 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -108,6 +108,13 @@ def _check_is_valid( except KeyError: raise InvalidTrackGeometryDataset(f"Missing entry for key {BASE_GEOMETRY}") + @property + def empty(self) -> bool: + return self._get_base_geometry().empty + + def _get_base_geometry(self) -> DataFrame: + return self._dataset[BASE_GEOMETRY] + @staticmethod def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: if len(dataset) == 0: diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index a100eff49..a7f2affa8 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -314,3 +314,18 @@ def test_as_dict(self, first_track: Track, second_track: Track) -> None: .to_dict(orient="index") } assert result == expected + + def test_get_base_geometry(self) -> None: + base_geometry = Mock() + geometry_dataset = PygeosTrackGeometryDataset({BASE_GEOMETRY: base_geometry}) + assert geometry_dataset._get_base_geometry() == base_geometry + + def test_empty_on_empty_dataset(self) -> None: + geometry_dataset = PygeosTrackGeometryDataset() + assert geometry_dataset.empty + + def test_empty_on_filled_dataset(self, first_track: Track) -> None: + track_dataset = create_track_dataset([first_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) + assert not geometry_dataset.empty From 5326f98760e378af89053f66188cf1f6e67afbb2 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:45:37 +0100 Subject: [PATCH 024/107] Add property to get track ids from tracks stored in PygeosTrackGeometryDataset --- .../track_geometry_store/pygeos_store.py | 9 +++++++++ .../track_geometry_store/test_pygeos_store.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 17922a8c0..a8c00b0ed 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -108,6 +108,15 @@ def _check_is_valid( except KeyError: raise InvalidTrackGeometryDataset(f"Missing entry for key {BASE_GEOMETRY}") + @property + def track_ids(self) -> set[str]: + """Get track ids of tracks stored in dataset. + + Returns: + set[str]: the track ids stored. + """ + return set(self._dataset[BASE_GEOMETRY].index) + @property def empty(self) -> bool: return self._get_base_geometry().empty diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index a7f2affa8..2e0cba6dd 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -329,3 +329,9 @@ def test_empty_on_filled_dataset(self, first_track: Track) -> None: geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) assert not geometry_dataset.empty + + def test_get_track_ids(self, first_track: Track) -> None: + track_dataset = create_track_dataset([first_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) + assert geometry_dataset.track_ids == {first_track.id.id} From 876387db30ad9542e23910821035cc5cc0e15eed Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:45:59 +0100 Subject: [PATCH 025/107] Add missing docstring --- .../plugin_datastore/track_geometry_store/pygeos_store.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index a8c00b0ed..36036dc78 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -119,6 +119,11 @@ def track_ids(self) -> set[str]: @property def empty(self) -> bool: + """Whether dataset is empty. + + Returns: + bool: True if dataset is empty, False otherwise. + """ return self._get_base_geometry().empty def _get_base_geometry(self) -> DataFrame: From 3e4cb492650ee07f22d1f2d2a2b43eb443c9f5a4 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:50:19 +0100 Subject: [PATCH 026/107] Remove unused method --- .../track_geometry_store/pygeos_store.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 36036dc78..5e622c88b 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -171,36 +171,6 @@ def _create_entries( } return entries - @staticmethod - def _create_track( - track: Track, offset: RelativeOffsetCoordinate | None = None - ) -> Geometry: - """Creates a prepared pygeos LINESTRING for given track. - - Args: - track (Track): the track. - offset (RelativeOffsetCoordinate | None): the offset to be applied to - geometry. Defaults to None. - - Returns: - Geometry: the prepared pygeos geometry. - """ - if offset: - geometry = linestrings( - [ - apply_offset( - detection.x, detection.y, detection.w, detection.h, offset - ) - for detection in track.detections - ] - ) - else: - geometry = linestrings( - [(detection.x, detection.y) for detection in track.detections] - ) - prepare(geometry) - return geometry - def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: raise NotImplementedError From b6579d3faff211ac6989860141d3ef2f86c31300 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:55:30 +0100 Subject: [PATCH 027/107] Implement adding tracks to existing track geometry dataset --- OTAnalytics/domain/track.py | 11 ++++ .../track_geometry_store/pygeos_store.py | 21 +++++- .../track_geometry_store/test_pygeos_store.py | 66 +++++++++++++++++-- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index f30e23002..c9cb6badc 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -683,6 +683,17 @@ def from_track_dataset(dataset: TrackDataset) -> "TrackGeometryDataset": @abstractmethod def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": + """Add tracks to existing dataset. + + Pre-existing tracks will be overwritten. + + Args: + tracks (Iterable[Track]): the tracks to add. + + Returns: + TrackGeometryDataset: the dataset with tracks added. + + """ raise NotImplementedError @abstractmethod diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 5e622c88b..58bd5403a 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -172,7 +172,26 @@ def _create_entries( return entries def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: - raise NotImplementedError + if self.empty: + new_entries = self._create_entries(tracks) + return PygeosTrackGeometryDataset( + { + BASE_GEOMETRY: DataFrame.from_dict( + new_entries, orient=ORIENTATION_INDEX + ) + } + ) + new_dataset = {} + existing_entries = self.as_dict() + for offset in existing_entries.keys(): + new_entries = self._create_entries(tracks, offset) + for track_id, entry in new_entries.items(): + existing_entries[offset][track_id] = entry + new_dataset[offset] = DataFrame.from_dict( + existing_entries[offset], orient=ORIENTATION_INDEX + ) + + return PygeosTrackGeometryDataset(new_dataset) def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: raise NotImplementedError diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 2e0cba6dd..dabbddf09 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -88,6 +88,41 @@ def first_track() -> Track: return track_builder.build_track() +@pytest.fixture +def first_track_merged(first_track: Track) -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("1") + track_builder.add_xy_bbox(6, 1) + track_builder.add_frame(6) + track_builder.add_second(6) + track_builder.append_detection() + + track_builder.add_xy_bbox(7, 1) + track_builder.add_frame(7) + track_builder.add_second(7) + track_builder.append_detection() + + track_builder.add_xy_bbox(8, 1) + track_builder.add_frame(8) + track_builder.add_second(8) + track_builder.append_detection() + + track_builder.add_xy_bbox(9, 1) + track_builder.add_frame(9) + track_builder.add_second(9) + track_builder.append_detection() + + track_builder.add_xy_bbox(10, 1) + track_builder.add_frame(10) + track_builder.add_second(10) + track_builder.append_detection() + track = track_builder.build_track() + merged_track = Mock() + merged_track.detections = first_track.detections + track.detections + merged_track.id = first_track.id + return merged_track + + @pytest.fixture def second_track() -> Track: track_builder = TrackBuilder() @@ -237,13 +272,32 @@ def test_from_track_dataset(self, simple_track: Track) -> None: expected = create_geometry_dataset_from([simple_track]) assert_track_geometry_dataset_equals(geometry_dataset, expected) - @pytest.mark.skip - def test_add_all(self, first_track: Track, second_track: Track) -> None: - track_dataset = create_track_dataset([first_track, second_track]) - geometry_dataset = PygeosTrackGeometryDataset({}) - geometry_dataset.add_all(track_dataset) + def test_add_all_on_empty_dataset( + self, first_track: Track, second_track: Track + ) -> None: + track_dataset = create_track_dataset([]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.add_all([first_track, second_track]) expected = create_geometry_dataset_from([first_track, second_track]) - assert_track_geometry_dataset_equals(geometry_dataset, expected) + assert_track_geometry_dataset_equals(result, expected) + + def test_add_all_on_filled_dataset( + self, first_track: Track, second_track: Track + ) -> None: + track_dataset = create_track_dataset([first_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.add_all([second_track]) + expected = create_geometry_dataset_from([first_track, second_track]) + assert_track_geometry_dataset_equals(result, expected) + + def test_add_all_merge_track( + self, first_track: Track, first_track_merged: Track, second_track: Track + ) -> None: + track_dataset = create_track_dataset([first_track, second_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.add_all([first_track_merged]) + expected = create_geometry_dataset_from([first_track_merged, second_track]) + assert_track_geometry_dataset_equals(result, expected) def test_intersection_points( self, From 3ae35d8ac8413a6f8ac48790c465490704135b59 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:49:22 +0100 Subject: [PATCH 028/107] Add __init__ files --- .../plugin_datastore/track_geometry_store/__init__.py | 9 +++++++++ .../plugin_datastore/track_geometry_store/__init__.py | 0 2 files changed, 9 insertions(+) create mode 100644 OTAnalytics/plugin_datastore/track_geometry_store/__init__.py create mode 100644 tests/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py b/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py new file mode 100644 index 000000000..f82154db7 --- /dev/null +++ b/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py @@ -0,0 +1,9 @@ +from typing import Callable + +from OTAnalytics.domain.track import TrackDataset, TrackGeometryDataset +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) + +TRACK_GEOMETRY_DATASET_FACTORY = PygeosTrackGeometryDataset.from_track_dataset +TRACK_GEOMETRY_FACTORY = Callable[[TrackDataset], TrackGeometryDataset] diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py new file mode 100644 index 000000000..e69de29bb From b329cf97cf3303d37443d437b25f7c337c92a9b5 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:04:49 +0100 Subject: [PATCH 029/107] Implement removing track geometries from dataset via TrackId --- OTAnalytics/domain/track.py | 10 ++++++--- .../track_geometry_store/pygeos_store.py | 13 ++++++------ .../track_geometry_store/test_pygeos_store.py | 21 +++++++++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index c9cb6badc..b48b00768 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -698,10 +698,14 @@ def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": @abstractmethod def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": - raise NotImplementedError + """Remove track geometries with given ids from dataset. - @abstractmethod - def clear(self) -> "TrackGeometryDataset": + Args: + ids (Iterable[TrackId]): the track geometries to remove. + + Returns: + TrackGeometryDataset: the dataset with tracks removed. + """ raise NotImplementedError @abstractmethod diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 58bd5403a..2f3911a73 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -76,10 +76,9 @@ def create_pygeos_track( class TrackGeometryEntry(TypedDict): + # TODO: Remove if not needed geometry: Geometry projection: list[float] - intersection: list - intersects: list class InvalidTrackGeometryDataset(Exception): @@ -194,10 +193,12 @@ def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: return PygeosTrackGeometryDataset(new_dataset) def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: - raise NotImplementedError - - def clear(self) -> TrackGeometryDataset: - raise NotImplementedError + updated = {} + for offset, geometry_df in self._dataset.items(): + updated[offset] = geometry_df.drop( + index=[track_id.id for track_id in ids], errors="ignore" + ) + return PygeosTrackGeometryDataset(updated) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: intersecting_tracks = set() diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index dabbddf09..6f225bd66 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -299,6 +299,27 @@ def test_add_all_merge_track( expected = create_geometry_dataset_from([first_track_merged, second_track]) assert_track_geometry_dataset_equals(result, expected) + def test_remove_from_filled_dataset( + self, first_track: Track, second_track: Track + ) -> None: + track_dataset = create_track_dataset([first_track, second_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.remove([first_track.id]) + expected = create_geometry_dataset_from([second_track]) + assert_track_geometry_dataset_equals(result, expected) + + def test_remove_from_empty_dataset(self, first_track: Track) -> None: + geometry_dataset = PygeosTrackGeometryDataset() + result = geometry_dataset.remove([first_track.id]) + assert_track_geometry_dataset_equals(result, PygeosTrackGeometryDataset()) + + def test_remove_missing(self, first_track: Track, second_track: Track) -> None: + track_dataset = create_track_dataset([first_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.remove([second_track.id]) + expected = create_geometry_dataset_from([first_track]) + assert_track_geometry_dataset_equals(result, expected) + def test_intersection_points( self, not_intersecting_track: Track, From fb689672f0db71a40353321eed990a19ab1b3ebb Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:00:32 +0100 Subject: [PATCH 030/107] Move default track geometry factory method to pandas track storage --- OTAnalytics/domain/track.py | 5 ++++- .../plugin_datastore/track_geometry_store/__init__.py | 9 --------- OTAnalytics/plugin_datastore/track_store.py | 10 ++++++---- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index b48b00768..f7c37aede 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Iterable, Iterator, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence from PIL import Image @@ -750,3 +750,6 @@ def contained_by_sections( coordinates contained by given sections. """ raise NotImplementedError + + +TRACK_GEOMETRY_FACTORY = Callable[[TrackDataset], TrackGeometryDataset] diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py b/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py index f82154db7..e69de29bb 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/__init__.py @@ -1,9 +0,0 @@ -from typing import Callable - -from OTAnalytics.domain.track import TrackDataset, TrackGeometryDataset -from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( - PygeosTrackGeometryDataset, -) - -TRACK_GEOMETRY_DATASET_FACTORY = PygeosTrackGeometryDataset.from_track_dataset -TRACK_GEOMETRY_FACTORY = Callable[[TrackDataset], TrackGeometryDataset] diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 06b30b382..e387229ba 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -12,15 +12,15 @@ from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( MIN_NUMBER_OF_DETECTIONS, + TRACK_GEOMETRY_FACTORY, Detection, IntersectionPoint, Track, TrackDataset, TrackId, ) -from OTAnalytics.plugin_datastore.track_geometry_store import ( - TRACK_GEOMETRY_DATASET_FACTORY, - TRACK_GEOMETRY_FACTORY, +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, ) @@ -149,7 +149,9 @@ def __init__( self, dataset: DataFrame = DataFrame(), calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, - track_geometry_factory: TRACK_GEOMETRY_FACTORY = TRACK_GEOMETRY_DATASET_FACTORY, + track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( + PygeosTrackGeometryDataset.from_track_dataset + ), ): self._dataset = dataset self._calculator = calculator From f98da54425b830ca960a8699be9d830c80849824 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:49:53 +0100 Subject: [PATCH 031/107] Implement contained by section method of TrackGeometryDataset This method is required by the area intersector later on. --- OTAnalytics/domain/track.py | 8 +- .../plugin_datastore/python_track_store.py | 17 ++++- .../track_geometry_store/pygeos_store.py | 47 ++++++++++-- OTAnalytics/plugin_datastore/track_store.py | 5 +- .../track_geometry_store/test_pygeos_store.py | 73 ++++++++++++++++++- 5 files changed, 131 insertions(+), 19 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index f7c37aede..fb943b279 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -441,14 +441,14 @@ def intersection_points( @abstractmethod def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: """Return whether track coordinates are contained by the given sections. Args: sections (Iterable[Section]): the sections. Returns: - dict[TrackId, tuple[SectionId, Sequence[bool]]]: boolean mask of track + dict[TrackId, dict[SectionId, Sequence[bool]]]: boolean mask of track coordinates contained by given sections. """ raise NotImplementedError @@ -739,14 +739,14 @@ def intersection_points( @abstractmethod def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: """Return whether track coordinates are contained by the given sections. Args: sections (Iterable[Section]): the sections. Returns: - dict[TrackId, tuple[SectionId, Sequence[bool]]]: boolean mask of track + dict[TrackId, dict[SectionId, Sequence[bool]]]: boolean mask of track coordinates contained by given sections. """ raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 3d15f481f..14fa617da 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -9,6 +9,7 @@ from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( + TRACK_GEOMETRY_FACTORY, Detection, IntersectionPoint, Track, @@ -17,6 +18,9 @@ TrackHasNoDetectionError, TrackId, ) +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) @dataclass(frozen=True) @@ -219,11 +223,16 @@ def __init__( self, values: Optional[dict[TrackId, Track]] = None, calculator: TrackClassificationCalculator = ByMaxConfidence(), + track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( + PygeosTrackGeometryDataset.from_track_dataset + ), ) -> None: if values is None: values = {} self._tracks = values self._calculator = calculator + self._track_geometry_factory = track_geometry_factory + self._track_geometry_dataset = track_geometry_factory(self) @staticmethod def from_list( @@ -312,14 +321,14 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": return PythonTrackDataset(filtered_tracks, self._calculator) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - raise NotImplementedError + return self._track_geometry_dataset.intersecting_tracks(sections) def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - raise NotImplementedError + return self._track_geometry_dataset.intersection_points(sections) def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: - raise NotImplementedError + ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + return self._track_geometry_dataset.contained_by_sections(sections) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 2f3911a73..13621f0ad 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -7,6 +7,7 @@ from pygeos import ( Geometry, apply, + contains, geometrycollections, get_coordinates, intersection, @@ -15,6 +16,7 @@ line_locate_point, linestrings, points, + polygons, prepare, ) @@ -39,14 +41,24 @@ ORIENTATION_INDEX: Literal["index"] = "index" -def sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: - return geometrycollections([section_to_pygeos(s) for s in sections]) +def line_sections_to_pygeos_multi(sections: Iterable[Section]) -> Geometry: + return geometrycollections([line_section_to_pygeos(s) for s in sections]) -def section_to_pygeos(section: Section) -> Geometry: +def line_section_to_pygeos(section: Section) -> Geometry: return linestrings([[(c.x, c.y) for c in section.get_coordinates()]]) +def area_sections_to_pygeos(sections: Iterable[Section]) -> list[Geometry]: + return [area_section_to_pygeos(s) for s in sections] + + +def area_section_to_pygeos(section: Section) -> Geometry: + geometry = polygons([[(c.x, c.y) for c in section.get_coordinates()]]) + prepare(geometry) + return geometry + + def create_pygeos_track( track: Track, offset: RelativeOffsetCoordinate = BASE_GEOMETRY ) -> Geometry: @@ -204,7 +216,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: intersecting_tracks = set() sections_grouped_by_offset = group_sections_by_offset(sections) for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = sections_to_pygeos_multi(_sections) + section_geoms = line_sections_to_pygeos_multi(_sections) track_df = self._get_track_geometries_for(offset) track_df[INTERSECTS] = ( @@ -225,7 +237,7 @@ def intersection_points( intersection_points = defaultdict(list) for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = sections_to_pygeos_multi(_sections) + section_geoms = line_sections_to_pygeos_multi(_sections) track_df = self._get_track_geometries_for(offset) track_df[INTERSECTIONS] = track_df[GEOMETRY].apply( @@ -297,8 +309,29 @@ def _next_event( def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: - raise NotImplementedError + ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + sections_grouped_by_offset = group_sections_by_offset(sections) + + contains_result: dict[TrackId, dict[SectionId, Sequence[bool]]] = defaultdict( + lambda: defaultdict(list) + ) + for offset, section_group in sections_grouped_by_offset.items(): + for _section in section_group: + section_geom = area_section_to_pygeos(_section) + + track_df = self._get_track_geometries_for(offset) + contains_masks = track_df[GEOMETRY].apply( + lambda line: { + _section.id: [ + contains(section_geom, points(p))[0] + for p in get_coordinates(line) + ] + } + ) + contains_masks.index = contains_masks.index.map(TrackId) + for track_id, entry in contains_masks.to_dict().items(): + contains_result[track_id].update(entry) + return contains_result def as_dict(self) -> dict: result = {} diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index e387229ba..bc45715b7 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -156,6 +156,7 @@ def __init__( self._dataset = dataset self._calculator = calculator self._track_geometry_factory = track_geometry_factory + # TODO: Re-use existing track geometries instead of creating new ones self._track_geometry_dataset = self._track_geometry_factory(self) @staticmethod @@ -269,8 +270,8 @@ def intersection_points( def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, tuple[SectionId, Sequence[bool]]]: - raise NotImplementedError + ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + return self._track_geometry_dataset.contained_by_sections(sections) def _assign_track_classification( diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 6f225bd66..584135788 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -6,7 +6,7 @@ from pygeos import Geometry, get_coordinates, line_locate_point, linestrings, points from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.section import LineSection, Section, SectionId +from OTAnalytics.domain.section import Area, LineSection, Section, SectionId from OTAnalytics.domain.track import ( IntersectionPoint, Track, @@ -158,7 +158,7 @@ def second_track() -> Track: @pytest.fixture def not_intersecting_track() -> Track: track_builder = TrackBuilder() - track_builder.add_track_id("3") + track_builder.add_track_id("not_intersecting_track") track_builder.add_xy_bbox(1, 10) track_builder.add_frame(1) track_builder.add_second(1) @@ -239,6 +239,44 @@ def third_section() -> Section: ) +@pytest.fixture +def area_section() -> Section: + name = "area" + coordinates = [ + Coordinate(1.5, 0.5), + Coordinate(3, 0.5), + Coordinate(3, 2), + Coordinate(1.5, 2), + Coordinate(1.5, 0.5), + ] + return Area( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + +@pytest.fixture +def not_intersecting_area_section() -> Section: + name = "not_intersecting_area" + coordinates = [ + Coordinate(20, 20), + Coordinate(30, 20), + Coordinate(30, 30), + Coordinate(20, 30), + Coordinate(20, 20), + ] + return Area( + SectionId(name), + name, + {EventType.SECTION_ENTER: RelativeOffsetCoordinate(0, 0)}, + {}, + coordinates, + ) + + def assert_track_geometry_dataset_equals( to_compare: TrackGeometryDataset, other: TrackGeometryDataset ) -> None: @@ -410,3 +448,34 @@ def test_get_track_ids(self, first_track: Track) -> None: geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) assert geometry_dataset.track_ids == {first_track.id.id} + + def test_contained_by_sections( + self, + first_track: Track, + second_track: Track, + not_intersecting_track: Track, + area_section: Section, + not_intersecting_area_section: Section, + ) -> None: + track_dataset = create_track_dataset( + [not_intersecting_track, first_track, second_track] + ) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + result = geometry_dataset.contained_by_sections( + [not_intersecting_area_section, area_section] + ) + expected = { + not_intersecting_track.id: { + not_intersecting_area_section.id: [False, False, False, False, False], + area_section.id: [False, False, False, False, False], + }, + first_track.id: { + not_intersecting_area_section.id: [False, False, False, False, False], + area_section.id: [False, True, False, False, False], + }, + second_track.id: { + not_intersecting_area_section.id: [False, False, False, False, False], + area_section.id: [False, True, False, False, False], + }, + } + assert result == expected From 000dbb2ffd6d463fb310d3a52098f8985472dd1c Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:21:47 +0100 Subject: [PATCH 032/107] Fix __len__ failing when PandasTrackDataset is empty --- OTAnalytics/plugin_datastore/track_store.py | 2 ++ tests/OTAnalytics/plugin_datastore/test_track_store.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index bc45715b7..accca30f2 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -245,6 +245,8 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: return new_batches def __len__(self) -> int: + if self._dataset.empty: + return 0 return len(self._dataset[track.TRACK_ID].unique()) def filter_by_min_detection_length(self, length: int) -> "TrackDataset": diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 4c34d4d70..ce4b340d4 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -174,8 +174,9 @@ def test_len(self) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") dataset = PandasTrackDataset.from_list([first_track, second_track]) - assert len(dataset) == 2 + empty_dataset = PandasTrackDataset.from_list([]) + assert len(empty_dataset) == 0 @pytest.mark.parametrize( "num_tracks,batches,expected_batches", [(10, 1, 1), (10, 4, 4), (3, 4, 3)] From 68ce03773d4be776a8d1ea8329174ca681cb0e2c Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:22:42 +0100 Subject: [PATCH 033/107] Don't store single detection tracks in TrackGeometryDataset --- .../track_geometry_store/pygeos_store.py | 3 +++ .../track_geometry_store/test_pygeos_store.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 13621f0ad..556c3ad1d 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -170,6 +170,9 @@ def _create_entries( """ entries = dict() for track in tracks: + if len(track.detections) < 2: + # Disregard single detection tracks + continue track_id = track.id.id geometry = create_pygeos_track(track, offset) projection = [ diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 584135788..28de14f76 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -187,6 +187,19 @@ def not_intersecting_track() -> Track: return track_builder.build_track() +@pytest.fixture +def single_detection_track() -> Track: + detection = Mock() + detection.x = 1 + detection.y = 1 + detection.w = 1 + detection.h = 1 + track = Mock() + track.id = TrackId("Single-Detection") + track.detections = [detection] + return track + + @pytest.fixture def not_intersecting_section() -> Section: name = "first" @@ -360,6 +373,7 @@ def test_remove_missing(self, first_track: Track, second_track: Track) -> None: def test_intersection_points( self, + single_detection_track: Track, not_intersecting_track: Track, first_track: Track, second_track: Track, @@ -375,7 +389,7 @@ def test_intersection_points( third_section, ] track_dataset = create_track_dataset( - [not_intersecting_track, first_track, second_track] + [single_detection_track, not_intersecting_track, first_track, second_track] ) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) result = geometry_dataset.intersection_points(list(sections)) @@ -394,6 +408,7 @@ def test_intersection_points( def test_intersecting_tracks( self, + single_detection_track: Track, not_intersecting_track: Track, first_track: Track, second_track: Track, @@ -409,7 +424,7 @@ def test_intersecting_tracks( third_section, ] track_dataset = create_track_dataset( - [not_intersecting_track, first_track, second_track] + [single_detection_track, not_intersecting_track, first_track, second_track] ) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) result = geometry_dataset.intersecting_tracks(list(sections)) From d7226bab624dbf4fe70802457615480bcc420cd1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:24:36 +0100 Subject: [PATCH 034/107] Use more generic argument type --- OTAnalytics/domain/track.py | 16 ++++++++++++---- .../plugin_datastore/python_track_store.py | 2 +- .../track_geometry_store/pygeos_store.py | 2 +- OTAnalytics/plugin_datastore/track_store.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index fb943b279..20503ccd0 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -411,11 +411,11 @@ def as_list(self) -> list[Track]: raise NotImplementedError @abstractmethod - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. Args: - sections (list[Section]): the list of sections to intersect. + sections (Iterable[Section]): the list of sections to intersect. Returns: set[TrackId]: the track ids intersecting the given sections. @@ -676,6 +676,14 @@ def reset(self) -> None: class TrackGeometryDataset(ABC): + """Dataset containing track geometries. + + Only tracks of size greater equal two are contained in the dataset. + Tracks of size less than two will not be contained in the dataset + since it is not possible to construct a track with less than two + coordinates. + """ + @staticmethod @abstractmethod def from_track_dataset(dataset: TrackDataset) -> "TrackGeometryDataset": @@ -709,11 +717,11 @@ def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": raise NotImplementedError @abstractmethod - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. Args: - sections (list[Section]): the list of sections to intersect. + sections (Iterable[Section]): the list of sections to intersect. Returns: set[TrackId]: the track ids intersecting the given sections. diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 14fa617da..6f69554c4 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -320,7 +320,7 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": } return PythonTrackDataset(filtered_tracks, self._calculator) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: return self._track_geometry_dataset.intersecting_tracks(sections) def intersection_points( diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 556c3ad1d..974c95674 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -215,7 +215,7 @@ def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: ) return PygeosTrackGeometryDataset(updated) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: intersecting_tracks = set() sections_grouped_by_offset = group_sections_by_offset(sections) for offset, _sections in sections_grouped_by_offset.items(): diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index accca30f2..ffcdf8137 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -262,7 +262,7 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": ] return PandasTrackDataset(filtered_dataset, self._calculator) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: return self._track_geometry_dataset.intersecting_tracks(sections) def intersection_points( From 9b77ec5043630a25553698873a390b62bad4946a Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:46:20 +0100 Subject: [PATCH 035/107] Add factory methods to return tracks as TrackDataset or list in GetAllTracks use case --- .../application/use_cases/track_repository.py | 7 +++++ .../use_cases/test_track_repository.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index f075ac99f..275ecfcde 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -25,6 +25,13 @@ def __init__(self, track_repository: TrackRepository) -> None: def __call__(self) -> Iterable[Track]: return self._track_repository.get_all() + def as_list(self) -> list[Track]: + return self.as_dataset().as_list() + + def as_dataset(self) -> TrackDataset: + tracks = self._track_repository.get_all() + return tracks.filter_by_min_detection_length(2) + class GetTracksWithoutSingleDetections: """Get tracks that have at least two detections. diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 8e93280e5..1ac1aec1e 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -56,6 +56,35 @@ def test_get_all_tracks(self, track_repository: Mock, tracks: TrackDataset) -> N assert result_tracks == tracks track_repository.get_all.assert_called_once() + def test_get_as_dataset(self) -> None: + expected_dataset = Mock() + dataset = Mock() + dataset.filter_by_min_detection_length.return_value = expected_dataset + track_repository = Mock() + track_repository.get_all.return_value = dataset + + get_tracks = GetAllTracks(track_repository) + result_dataset = get_tracks.as_dataset() + assert result_dataset == expected_dataset + track_repository.get_all.assert_called_once() + dataset.filter_by_min_detection_length.assert_called_once_with(2) + + def test_get_as_list(self) -> None: + track_repository = Mock() + + get_tracks = GetAllTracks(track_repository) + with patch.object(GetAllTracks, "as_dataset") as mock_as_dataset: + expected_list = Mock() + filtered_dataset = Mock() + filtered_dataset.as_list.return_value = expected_list + + mock_as_dataset.return_value = filtered_dataset + result = get_tracks.as_list() + + assert result == expected_list + mock_as_dataset.assert_called_once() + filtered_dataset.as_list.assert_called_once() + class TestGetAllTrackIds: def test_get_all_tracks(self, track_repository: Mock) -> None: From 82de9eb037162e5c2007fb8f8e86521148aff0b1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:05:40 +0100 Subject: [PATCH 036/107] Use geometry operations of TrackDataset in TracksIntersectingSections use case --- OTAnalytics/application/geometry.py | 29 +------ .../plugin_intersect/simple_intersect.py | 58 ++++---------- OTAnalytics/plugin_ui/main_application.py | 16 ++-- .../plugin_ui/visualization/visualization.py | 8 +- .../OTAnalytics/application/test_geometry.py | 76 ------------------- .../use_cases/test_track_repository.py | 8 +- .../plugin_intersect/test_intersect.py | 38 ++-------- 7 files changed, 34 insertions(+), 199 deletions(-) delete mode 100644 tests/OTAnalytics/application/test_geometry.py diff --git a/OTAnalytics/application/geometry.py b/OTAnalytics/application/geometry.py index 600c58ad4..307f01831 100644 --- a/OTAnalytics/application/geometry.py +++ b/OTAnalytics/application/geometry.py @@ -2,36 +2,9 @@ from functools import singledispatchmethod from typing import Generic, Iterable, TypeVar -from OTAnalytics.domain.geometry import ( - Coordinate, - Line, - Polygon, - RelativeOffsetCoordinate, -) -from OTAnalytics.domain.section import Section +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.track import Track - -class SectionGeometryBuilder: - def build_as_line(self, section: Section) -> Line: - return Line(section.get_coordinates()) - - def build_as_polygon(self, section: Section) -> Polygon: - return Polygon(section.get_coordinates()) - - -class TrackGeometryBuilder: - def build(self, track: Track, offset: RelativeOffsetCoordinate) -> Line: - coordinates = [ - Coordinate( - detection.x + offset.x * detection.w, - detection.y + offset.y * detection.h, - ) - for detection in track.detections - ] - return Line(coordinates) - - LINE = TypeVar("LINE") AREA = TypeVar("AREA") diff --git a/OTAnalytics/plugin_intersect/simple_intersect.py b/OTAnalytics/plugin_intersect/simple_intersect.py index 4cc1c205d..aaa460a38 100644 --- a/OTAnalytics/plugin_intersect/simple_intersect.py +++ b/OTAnalytics/plugin_intersect/simple_intersect.py @@ -2,57 +2,27 @@ from typing import Iterable from OTAnalytics.application.analysis.intersect import TracksIntersectingSections -from OTAnalytics.application.geometry import ( - SectionGeometryBuilder, - TrackGeometryBuilder, -) -from OTAnalytics.application.use_cases.track_repository import ( - GetTracksWithoutSingleDetections, -) -from OTAnalytics.domain.event import EventType -from OTAnalytics.domain.intersect import IntersectImplementation +from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track import TrackId class SimpleTracksIntersectingSections(TracksIntersectingSections): - def __init__( - self, - get_tracks: GetTracksWithoutSingleDetections, - intersect_implementation: IntersectImplementation, - track_geometry_builder: TrackGeometryBuilder = TrackGeometryBuilder(), - section_geometry_builder: SectionGeometryBuilder = SectionGeometryBuilder(), - ): + def __init__(self, get_tracks: GetAllTracks): self._get_tracks = get_tracks - self._intersect_implementation = intersect_implementation - self._track_geometry_builder = track_geometry_builder - self._section_geometry_builder = section_geometry_builder def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: - tracks = self._get_tracks() - return self._intersect(tracks, sections) + return self.intersect(sections) - def _intersect( - self, tracks: Iterable[Track], sections: Iterable[Section] - ) -> dict[SectionId, set[TrackId]]: + def intersect(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: + track_dataset = self._get_tracks.as_dataset() + result = defaultdict(set) + total_tracks_intersected = 0 print("Number of intersecting tracks per section") - all_track_ids: dict[SectionId, set[TrackId]] = defaultdict(set) for section in sections: - track_ids = { - track.id - for track in tracks - if self._track_intersects_section(track, section) - } - print(f"{section.name}: {len(track_ids)} tracks") - all_track_ids[section.id].update(track_ids) - - print(f"All sections: {len(all_track_ids)} tracks") - return all_track_ids - - def _track_intersects_section(self, track: Track, section: Section) -> bool: - section_offset = section.get_offset(EventType.SECTION_ENTER) - track_as_geom = self._track_geometry_builder.build(track, section_offset) - section_as_geom = self._section_geometry_builder.build_as_line(section) - return self._intersect_implementation.line_intersects_line( - track_as_geom, section_as_geom - ) + result[section.id].update(track_dataset.intersecting_tracks([section])) + num_tracks_intersected = len(result[section.id]) + total_tracks_intersected += num_tracks_intersected + print(f"{section.name}: {num_tracks_intersected} tracks") + print(f"All sections: {total_tracks_intersected} tracks") + return result diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 8f7042994..b95c32473 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -81,6 +81,7 @@ ClearAllTracks, GetAllTrackFiles, GetAllTrackIds, + GetAllTracks, GetTracksFromIds, GetTracksWithoutSingleDetections, RemoveTracks, @@ -93,7 +94,6 @@ from OTAnalytics.domain.event import EventRepository, SceneEventBuilder from OTAnalytics.domain.filter import FilterElementSettingRestorer from OTAnalytics.domain.flow import FlowRepository -from OTAnalytics.domain.intersect import IntersectImplementation from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionRepository from OTAnalytics.domain.track import TrackFileRepository, TrackRepository @@ -105,7 +105,6 @@ from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( ShapelyRunIntersect, ) -from OTAnalytics.plugin_intersect.shapely.intersect import ShapelyIntersector from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTrackSegmentBuilder, @@ -261,6 +260,7 @@ def start_gui(self) -> None: ) get_all_track_files = self._create_get_all_track_files(track_file_repository) + get_all_tracks = GetAllTracks(track_repository) get_tracks_without_single_detections = GetTracksWithoutSingleDetections( track_repository ) @@ -342,8 +342,7 @@ def start_gui(self) -> None: clear_repositories, reset_project_config, track_view_state ) tracks_intersecting_sections = self._create_tracks_intersecting_sections( - GetTracksWithoutSingleDetections(track_repository), - ShapelyIntersector(), + get_all_tracks ) cut_tracks_intersecting_section = self._create_cut_tracks_intersecting_section( get_sections_bv_id, @@ -454,6 +453,7 @@ def start_cli(self, cli_args: CliArguments) -> None: get_tracks_without_single_detections = GetTracksWithoutSingleDetections( track_repository ) + get_all_tracks = GetAllTracks(track_repository) get_all_track_ids = GetAllTrackIds(track_repository) clear_all_events = ClearAllEvents(event_repository) create_events = self._create_use_case_create_events( @@ -464,8 +464,7 @@ def start_cli(self, cli_args: CliArguments) -> None: cli_args.num_processes, ) tracks_intersecting_sections = self._create_tracks_intersecting_sections( - GetTracksWithoutSingleDetections(track_repository), - ShapelyIntersector(), + get_all_tracks ) cut_tracks = self._create_cut_tracks_intersecting_section( GetSectionsById(section_repository), @@ -712,10 +711,9 @@ def _create_use_case_create_events( @staticmethod def _create_tracks_intersecting_sections( - get_tracks: GetTracksWithoutSingleDetections, - intersect_implementation: IntersectImplementation, + get_tracks: GetAllTracks, ) -> TracksIntersectingSections: - return SimpleTracksIntersectingSections(get_tracks, intersect_implementation) + return SimpleTracksIntersectingSections(get_tracks) @staticmethod def _create_use_case_load_otflow( diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index ff457d27d..45deb611f 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -19,15 +19,12 @@ TracksOverlapOccurrenceWindow, ) from OTAnalytics.application.use_cases.section_repository import GetSectionsById -from OTAnalytics.application.use_cases.track_repository import ( - GetTracksWithoutSingleDetections, -) +from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.flow import FlowId from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionId from OTAnalytics.domain.track import TrackIdProvider from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder -from OTAnalytics.plugin_intersect.shapely.intersect import ShapelyIntersector from OTAnalytics.plugin_intersect.simple_intersect import ( SimpleTracksIntersectingSections, ) @@ -576,6 +573,5 @@ def _create_dataframe_filter_builder(self) -> DataFrameFilterBuilder: # TODO duplicate to main_application.py def _create_tracks_intersecting_sections(self) -> TracksIntersectingSections: return SimpleTracksIntersectingSections( - GetTracksWithoutSingleDetections(self._track_repository), - ShapelyIntersector(), + GetAllTracks(self._track_repository), ) diff --git a/tests/OTAnalytics/application/test_geometry.py b/tests/OTAnalytics/application/test_geometry.py deleted file mode 100644 index 8304bbb50..000000000 --- a/tests/OTAnalytics/application/test_geometry.py +++ /dev/null @@ -1,76 +0,0 @@ -from unittest.mock import Mock - -import pytest - -from OTAnalytics.application.geometry import ( - SectionGeometryBuilder, - TrackGeometryBuilder, -) -from OTAnalytics.domain.geometry import ( - Coordinate, - Line, - Polygon, - RelativeOffsetCoordinate, -) -from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import Detection, Track - - -class TestTrackGeometryBuilder: - def test_build(self) -> None: - detection_1 = Mock(spec=Detection) - detection_1.x = 0 - detection_1.y = 0 - detection_1.w = 1 - detection_1.h = 1 - - detection_2 = Mock(spec=Detection) - detection_2.x = 1 - detection_2.y = 1 - detection_2.w = 1 - detection_2.h = 1 - - track = Mock(spec=Track) - track.detections = [detection_1, detection_2] - builder = TrackGeometryBuilder() - result = builder.build(track, RelativeOffsetCoordinate(0.5, 0.5)) - - assert result == Line([Coordinate(0.5, 0.5), Coordinate(1.5, 1.5)]) - - -class TestSectionGeometryBuilder: - @pytest.fixture - def polygon_coordinates(self) -> list[Coordinate]: - return [ - Coordinate(0, 0), - Coordinate(1, 0), - Coordinate(1, 1), - Coordinate(0, 1), - Coordinate(0, 0), - ] - - @pytest.fixture - def section(self, polygon_coordinates: list[Coordinate]) -> Mock: - section = Mock(spec=Section) - section.get_coordinates.return_value = polygon_coordinates - return section - - def test_build_as_line( - self, section: Mock, polygon_coordinates: list[Coordinate] - ) -> None: - builder = SectionGeometryBuilder() - result = builder.build_as_line(section) - - assert isinstance(result, Line) - assert result.coordinates == polygon_coordinates - section.get_coordinates.assert_called_once() - - def test_build_as_polygon( - self, section: Mock, polygon_coordinates: list[Coordinate] - ) -> None: - builder = SectionGeometryBuilder() - result = builder.build_as_polygon(section) - - assert isinstance(result, Polygon) - assert result.coordinates == polygon_coordinates - section.get_coordinates.assert_called_once() diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 1ac1aec1e..407db00dd 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -1,5 +1,5 @@ from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest @@ -21,12 +21,14 @@ TrackId, TrackRepository, ) -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset @pytest.fixture def tracks() -> TrackDataset: - return PythonTrackDataset.from_list([Mock(spec=Track), Mock(spec=Track)]) + tracks = [Mock(spec=Track), Mock(spec=Track)] + dataset = MagicMock(spec=TrackDataset) + dataset.__iter__.return_value = tracks + return dataset @pytest.fixture diff --git a/tests/OTAnalytics/plugin_intersect/test_intersect.py b/tests/OTAnalytics/plugin_intersect/test_intersect.py index de6dd1af5..b2553279e 100644 --- a/tests/OTAnalytics/plugin_intersect/test_intersect.py +++ b/tests/OTAnalytics/plugin_intersect/test_intersect.py @@ -2,14 +2,6 @@ import pytest -from OTAnalytics.application.geometry import ( - SectionGeometryBuilder, - TrackGeometryBuilder, -) -from OTAnalytics.application.use_cases.track_repository import GetAllTracks -from OTAnalytics.domain.event import EventType -from OTAnalytics.domain.geometry import Line, RelativeOffsetCoordinate -from OTAnalytics.domain.intersect import IntersectImplementation from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track from OTAnalytics.plugin_intersect.simple_intersect import ( @@ -57,39 +49,19 @@ def track(track_builder: TrackBuilder) -> Track: class TestSimpleTracksIntersectingSections: def test_tracks_intersecting_sections(self, track: Track) -> None: - get_all_tracks = Mock(spec=GetAllTracks) - get_all_tracks.return_value = [track] + track_dataset = Mock() + track_dataset.intersecting_tracks.return_value = {track.id} + get_all_tracks = Mock() + get_all_tracks.as_dataset.return_value = track_dataset section = Mock(spec=Section) section.id = SectionId("section-1") - offset = RelativeOffsetCoordinate(0, 0) - section.get_offset.return_value = offset section.name = "south" - intersect_implementation = Mock(spec=IntersectImplementation) - intersect_implementation.line_intersects_line.return_value = True - - section_geom = Mock(spec=Line) - track_geom = Mock(spec=Line) - - track_geometry_builder = Mock(spec=TrackGeometryBuilder) - track_geometry_builder.build.return_value = track_geom - section_geometry_builder = Mock(spec=SectionGeometryBuilder) - section_geometry_builder.build_as_line.return_value = section_geom - tracks_intersecting_sections = SimpleTracksIntersectingSections( get_all_tracks, - intersect_implementation, - track_geometry_builder, - section_geometry_builder, ) intersecting = tracks_intersecting_sections([section]) assert intersecting == {section.id: {track.id}} - get_all_tracks.assert_called_once() - section.get_offset.assert_called_once_with(EventType.SECTION_ENTER) - track_geometry_builder.build.assert_called_once_with(track, offset) - section_geometry_builder.build_as_line.assert_called_once_with(section) - intersect_implementation.line_intersects_line.assert_called_once_with( - track_geom, section_geom - ) + get_all_tracks.as_dataset.assert_called_once() From 9a26e2bd00fe10dcc3ef2352fa359a8a698d69e7 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 22 Nov 2023 09:58:26 +0100 Subject: [PATCH 037/107] Add function to separate sections by type --- .../shapely/create_intersection_events.py | 20 ++++++++++++++ .../test_create_intersection_events.py | 27 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index d9749a27b..f78ed6a71 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -295,3 +295,23 @@ def _create_events(tracks: Iterable[Track], sections: Iterable[Section]) -> list ) events.extend(create_intersection_events.create()) return events + + +def separate_sections( + sections: Iterable[Section], +) -> tuple[Iterable[LineSection], Iterable[Area]]: + line_sections = [] + area_sections = [] + for section in sections: + if isinstance(section, LineSection): + line_sections.append(section) + elif isinstance(section, Area): + area_sections.append(section) + else: + raise TypeError( + "Unable to separate section. " + f"Unknown section type for section {section.name} " + f"with type {type(section)}" + ) + + return line_sections, area_sections diff --git a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py index 85ac99c95..e6575dfc5 100644 --- a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py +++ b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py @@ -23,6 +23,7 @@ ShapelyIntersectAreaByTrackPoints, ShapelyIntersectBySmallestTrackSegments, ShapelyTrackLookupTable, + separate_sections, ) from tests.conftest import TrackBuilder @@ -510,3 +511,29 @@ def test_intersect( ) test_case.assert_valid(result_events, event_builder) + + +def test_separate_sections_with_valid_args() -> None: + first_line_section = Mock(spec=LineSection) + second_line_section = Mock(spec=LineSection) + + first_area_section = Mock(spec=Area) + second_area_section = Mock(spec=Area) + line_sections, area_sections = separate_sections( + [ + first_line_section, + first_area_section, + second_line_section, + second_area_section, + ] + ) + assert line_sections == [first_line_section, second_line_section] + assert area_sections == [first_area_section, second_area_section] + + +def test_separate_sections_with_invalid_args() -> None: + section = Mock(spec=LineSection) + invalid_section = Mock() + + with pytest.raises(TypeError): + separate_sections([section, invalid_section]) From 91be3a45595be44bb4eacb35a8e6ae2c16c33884 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 22 Nov 2023 10:38:47 +0100 Subject: [PATCH 038/107] Enable EventBuilder interface to add section id --- OTAnalytics/domain/event.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index 952cab983..b24df9d76 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -136,6 +136,7 @@ def __init__(self) -> None: self.event_type: Optional[EventType] = None self.direction_vector: Optional[DirectionVector2D] = None self.event_coordinate: Optional[ImageCoordinate] = None + self.section_id: Optional[SectionId] = None @abstractmethod def create_event(self, detection: Detection) -> Event: @@ -173,7 +174,7 @@ def extract_hostname(name: str) -> str: raise ImproperFormattedFilename(f"Could not parse {name}. Hostname is missing.") def add_road_user_type(self, road_user_type: str) -> None: - """Add a road user type to add to the event to be build. + """Add a road user type to add to the event to be built. Args: road_user_type (str): the road user type @@ -181,7 +182,7 @@ def add_road_user_type(self, road_user_type: str) -> None: self.road_user_type = road_user_type def add_event_type(self, event_type: EventType) -> None: - """Add an event type to add to the event to be build. + """Add an event type to add to the event to be built. Args: event_type (EventType): the event type @@ -189,7 +190,7 @@ def add_event_type(self, event_type: EventType) -> None: self.event_type = event_type def add_direction_vector(self, vector: DirectionVector2D) -> None: - """Add direction vector to add to the event to be build. + """Add direction vector to add to the event to be built. Args: vector (DirectionVector2D): the direction vector to be build @@ -197,7 +198,7 @@ def add_direction_vector(self, vector: DirectionVector2D) -> None: self.direction_vector = vector def add_event_coordinate(self, x: float, y: float) -> None: - """Add event coordinate to the event to be build. + """Add event coordinate to the event to be built. Args: x (float): the x component coordinate @@ -205,6 +206,14 @@ def add_event_coordinate(self, x: float, y: float) -> None: """ self.event_coordinate = ImageCoordinate(x, y) + def add_section_id(self, section_id: SectionId) -> None: + """Add a section id to add to the event to be built. + + Args: + section_id (SectionId): the section id + """ + self.section_id = section_id + class SectionEventBuilder(EventBuilder): """A builder to build section events.""" @@ -213,16 +222,8 @@ def __init__(self) -> None: super().__init__() self.section_id: Optional[SectionId] = None - def add_section_id(self, section_id: SectionId) -> None: - """Add a section id to add to the event to be build. - - Args: - section_id (SectionId): the section id - """ - self.section_id = section_id - def create_event(self, detection: Detection) -> Event: - """Creates an section event with the information stored in a detection. + """Creates a section event with the information stored in a detection. Args: detection (Detection): the detection holding the information From aa832a36e39b59fd9113c17689fff45da1731bd2 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:23:39 +0100 Subject: [PATCH 039/107] Return only tracks that have coordinates contained by section --- OTAnalytics/domain/track.py | 10 +++--- .../plugin_datastore/python_track_store.py | 2 +- .../track_geometry_store/pygeos_store.py | 31 ++++++++++--------- OTAnalytics/plugin_datastore/track_store.py | 2 +- .../track_geometry_store/test_pygeos_store.py | 18 ++++------- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 20503ccd0..945c49312 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -441,14 +441,14 @@ def intersection_points( @abstractmethod def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: """Return whether track coordinates are contained by the given sections. Args: sections (Iterable[Section]): the sections. Returns: - dict[TrackId, dict[SectionId, Sequence[bool]]]: boolean mask of track + dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask of track coordinates contained by given sections. """ raise NotImplementedError @@ -747,15 +747,15 @@ def intersection_points( @abstractmethod def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: """Return whether track coordinates are contained by the given sections. Args: sections (Iterable[Section]): the sections. Returns: - dict[TrackId, dict[SectionId, Sequence[bool]]]: boolean mask of track - coordinates contained by given sections. + dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask + of track coordinates contained by given sections. """ raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 6f69554c4..8ee55c86e 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -330,5 +330,5 @@ def intersection_points( def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: return self._track_geometry_dataset.contained_by_sections(sections) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 974c95674..fd66d1a0d 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -1,7 +1,7 @@ from bisect import bisect from collections import defaultdict from itertools import chain -from typing import Any, Iterable, Literal, Sequence, TypedDict +from typing import Any, Iterable, Literal, TypedDict from pandas import DataFrame from pygeos import ( @@ -312,28 +312,31 @@ def _next_event( def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: sections_grouped_by_offset = group_sections_by_offset(sections) - contains_result: dict[TrackId, dict[SectionId, Sequence[bool]]] = defaultdict( - lambda: defaultdict(list) - ) + contains_result: dict[ + TrackId, list[tuple[SectionId, list[bool]]] + ] = defaultdict(list) for offset, section_group in sections_grouped_by_offset.items(): for _section in section_group: section_geom = area_section_to_pygeos(_section) track_df = self._get_track_geometries_for(offset) contains_masks = track_df[GEOMETRY].apply( - lambda line: { - _section.id: [ - contains(section_geom, points(p))[0] - for p in get_coordinates(line) - ] - } + lambda line: [ + contains(section_geom, points(p))[0] + for p in get_coordinates(line) + ] ) - contains_masks.index = contains_masks.index.map(TrackId) - for track_id, entry in contains_masks.to_dict().items(): - contains_result[track_id].update(entry) + tracks_contained = contains_masks[contains_masks.map(any)] + + if tracks_contained.empty: + continue + + tracks_contained.index = tracks_contained.index.map(TrackId) + for track_id, contains_mask in tracks_contained.to_dict().items(): + contains_result[track_id].append((_section.id, contains_mask)) return contains_result def as_dict(self) -> dict: diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index ffcdf8137..2efce28cc 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -272,7 +272,7 @@ def intersection_points( def contained_by_sections( self, sections: Iterable[Section] - ) -> dict[TrackId, dict[SectionId, Sequence[bool]]]: + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: return self._track_geometry_dataset.contained_by_sections(sections) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 28de14f76..aaca7e83d 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -480,17 +480,11 @@ def test_contained_by_sections( [not_intersecting_area_section, area_section] ) expected = { - not_intersecting_track.id: { - not_intersecting_area_section.id: [False, False, False, False, False], - area_section.id: [False, False, False, False, False], - }, - first_track.id: { - not_intersecting_area_section.id: [False, False, False, False, False], - area_section.id: [False, True, False, False, False], - }, - second_track.id: { - not_intersecting_area_section.id: [False, False, False, False, False], - area_section.id: [False, True, False, False, False], - }, + first_track.id: [ + (area_section.id, [False, True, False, False, False]), + ], + second_track.id: [ + (area_section.id, [False, True, False, False, False]), + ], } assert result == expected From f61b3564e7572c6b6b0e34ceab908baadce4cbca Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:35:50 +0100 Subject: [PATCH 040/107] Move responsibility to group sections by offset from TrackGeometryDataset to user --- OTAnalytics/domain/track.py | 30 ++-- .../plugin_datastore/python_track_store.py | 15 +- .../track_geometry_store/pygeos_store.py | 134 ++++++++---------- OTAnalytics/plugin_datastore/track_store.py | 15 +- .../track_geometry_store/test_pygeos_store.py | 45 +++++- 5 files changed, 142 insertions(+), 97 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 945c49312..115e2bb05 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -411,11 +411,14 @@ def as_list(self) -> list[Track]: raise NotImplementedError @abstractmethod - def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. Args: - sections (Iterable[Section]): the list of sections to intersect. + sections (list[Section]): the list of sections to intersect. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: set[TrackId]: the track ids intersecting the given sections. @@ -424,7 +427,7 @@ def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: @abstractmethod def intersection_points( - self, sections: list[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: """ Return the intersection points resulting from the tracks and the @@ -432,6 +435,7 @@ def intersection_points( Args: sections (list[Section]): the sections to intersect with. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId]]]: the intersection points. @@ -440,12 +444,13 @@ def intersection_points( @abstractmethod def contained_by_sections( - self, sections: Iterable[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: """Return whether track coordinates are contained by the given sections. Args: - sections (Iterable[Section]): the sections. + sections (list[Section]): the sections. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask of track @@ -717,11 +722,14 @@ def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": raise NotImplementedError @abstractmethod - def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. Args: - sections (Iterable[Section]): the list of sections to intersect. + sections (list[Section]): the list of sections to intersect. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: set[TrackId]: the track ids intersecting the given sections. @@ -730,7 +738,7 @@ def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: @abstractmethod def intersection_points( - self, sections: list[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: """ Return the intersection points resulting from the tracks and the @@ -738,6 +746,7 @@ def intersection_points( Args: sections (list[Section]): the sections to intersect with. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId]]]: the intersection points. @@ -746,12 +755,13 @@ def intersection_points( @abstractmethod def contained_by_sections( - self, sections: Iterable[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: """Return whether track coordinates are contained by the given sections. Args: - sections (Iterable[Section]): the sections. + sections (list[Section]): the sections. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 8ee55c86e..840027ce8 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -7,6 +7,7 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain.common import DataclassValidation +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, @@ -320,15 +321,17 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": } return PythonTrackDataset(filtered_tracks, self._calculator) - def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: - return self._track_geometry_dataset.intersecting_tracks(sections) + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: + return self._track_geometry_dataset.intersecting_tracks(sections, offset) def intersection_points( - self, sections: list[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._track_geometry_dataset.intersection_points(sections) + return self._track_geometry_dataset.intersection_points(sections, offset) def contained_by_sections( - self, sections: Iterable[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - return self._track_geometry_dataset.contained_by_sections(sections) + return self._track_geometry_dataset.contained_by_sections(sections, offset) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index fd66d1a0d..40853b55f 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -20,7 +20,6 @@ prepare, ) -from OTAnalytics.application.analysis.intersect import group_sections_by_offset from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( @@ -215,63 +214,58 @@ def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: ) return PygeosTrackGeometryDataset(updated) - def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: intersecting_tracks = set() - sections_grouped_by_offset = group_sections_by_offset(sections) - for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = line_sections_to_pygeos_multi(_sections) - track_df = self._get_track_geometries_for(offset) - - track_df[INTERSECTS] = ( - track_df[GEOMETRY] - .apply(lambda line: intersects(line, section_geoms)) - .map(any) - .astype(bool) - ) - track_ids = [TrackId(_id) for _id in track_df[track_df[INTERSECTS]].index] - intersecting_tracks.update(track_ids) + section_geoms = line_sections_to_pygeos_multi(sections) + track_df = self._get_track_geometries_for(offset) + + track_df[INTERSECTS] = ( + track_df[GEOMETRY] + .apply(lambda line: intersects(line, section_geoms)) + .map(any) + .astype(bool) + ) + track_ids = [TrackId(_id) for _id in track_df[track_df[INTERSECTS]].index] + intersecting_tracks.update(track_ids) return intersecting_tracks def intersection_points( - self, sections: list[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - sections_grouped_by_offset = group_sections_by_offset(sections) - intersection_points = defaultdict(list) - for offset, _sections in sections_grouped_by_offset.items(): - section_geoms = line_sections_to_pygeos_multi(_sections) - track_df = self._get_track_geometries_for(offset) - - track_df[INTERSECTIONS] = track_df[GEOMETRY].apply( - lambda line: [ - (_sections[index].id, ip) - for index, ip in enumerate(intersection(line, section_geoms)) - if not is_empty(ip) - ] - ) - intersections = ( - track_df[track_df[INTERSECTIONS].apply(lambda i: len(i) > 0)] - .apply( - lambda r: [ - self._next_event( - r.name, # the track id (track ids is used as df index) - _section_id, - r[GEOMETRY], - points(p), - r[PROJECTION], - ) - for _section_id, ip in r[INTERSECTIONS] - for p in get_coordinates(ip) - ], - axis=1, - ) - .values + section_geoms = line_sections_to_pygeos_multi(sections) + track_df = self._get_track_geometries_for(offset) + + track_df[INTERSECTIONS] = track_df[GEOMETRY].apply( + lambda line: [ + (sections[index].id, ip) + for index, ip in enumerate(intersection(line, section_geoms)) + if not is_empty(ip) + ] + ) + intersections = ( + track_df[track_df[INTERSECTIONS].apply(lambda i: len(i) > 0)] + .apply( + lambda r: [ + self._next_event( + r.name, # the track id (track ids is used as df index) + _section_id, + r[GEOMETRY], + points(p), + r[PROJECTION], + ) + for _section_id, ip in r[INTERSECTIONS] + for p in get_coordinates(ip) + ], + axis=1, ) - for _id, section_id, intersection_point in chain.from_iterable( - intersections - ): - intersection_points[_id].append((section_id, intersection_point)) + .values + ) + for _id, section_id, intersection_point in chain.from_iterable(intersections): + intersection_points[_id].append((section_id, intersection_point)) return intersection_points @@ -311,32 +305,28 @@ def _next_event( ) def contained_by_sections( - self, sections: Iterable[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - sections_grouped_by_offset = group_sections_by_offset(sections) - contains_result: dict[ TrackId, list[tuple[SectionId, list[bool]]] ] = defaultdict(list) - for offset, section_group in sections_grouped_by_offset.items(): - for _section in section_group: - section_geom = area_section_to_pygeos(_section) - - track_df = self._get_track_geometries_for(offset) - contains_masks = track_df[GEOMETRY].apply( - lambda line: [ - contains(section_geom, points(p))[0] - for p in get_coordinates(line) - ] - ) - tracks_contained = contains_masks[contains_masks.map(any)] - - if tracks_contained.empty: - continue - - tracks_contained.index = tracks_contained.index.map(TrackId) - for track_id, contains_mask in tracks_contained.to_dict().items(): - contains_result[track_id].append((_section.id, contains_mask)) + for _section in sections: + section_geom = area_section_to_pygeos(_section) + + track_df = self._get_track_geometries_for(offset) + contains_masks = track_df[GEOMETRY].apply( + lambda line: [ + contains(section_geom, points(p))[0] for p in get_coordinates(line) + ] + ) + tracks_contained = contains_masks[contains_masks.map(any)] + + if tracks_contained.empty: + continue + + tracks_contained.index = tracks_contained.index.map(TrackId) + for track_id, contains_mask in tracks_contained.to_dict().items(): + contains_result[track_id].append((_section.id, contains_mask)) return contains_result def as_dict(self) -> dict: diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 2efce28cc..63cd39609 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -9,6 +9,7 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( MIN_NUMBER_OF_DETECTIONS, @@ -262,18 +263,20 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": ] return PandasTrackDataset(filtered_dataset, self._calculator) - def intersecting_tracks(self, sections: Iterable[Section]) -> set[TrackId]: - return self._track_geometry_dataset.intersecting_tracks(sections) + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: + return self._track_geometry_dataset.intersecting_tracks(sections, offset) def intersection_points( - self, sections: list[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._track_geometry_dataset.intersection_points(sections) + return self._track_geometry_dataset.intersection_points(sections, offset) def contained_by_sections( - self, sections: Iterable[Section] + self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - return self._track_geometry_dataset.contained_by_sections(sections) + return self._track_geometry_dataset.contained_by_sections(sections, offset) def _assign_track_classification( diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index aaca7e83d..b9adf0d57 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -1,9 +1,11 @@ +from pathlib import Path from typing import Iterable from unittest.mock import MagicMock, Mock import pytest from pandas import DataFrame from pygeos import Geometry, get_coordinates, line_locate_point, linestrings, points +from pytest_benchmark.fixture import BenchmarkFixture from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Area, LineSection, Section, SectionId @@ -23,6 +25,9 @@ TRACK_ID, PygeosTrackGeometryDataset, ) +from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence +from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser, OttrkParser +from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser from tests.conftest import TrackBuilder @@ -392,7 +397,7 @@ def test_intersection_points( [single_detection_track, not_intersecting_track, first_track, second_track] ) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) - result = geometry_dataset.intersection_points(list(sections)) + result = geometry_dataset.intersection_points(sections, BASE_GEOMETRY) assert result == { first_track.id: [ (first_section.id, IntersectionPoint(1)), @@ -427,7 +432,7 @@ def test_intersecting_tracks( [single_detection_track, not_intersecting_track, first_track, second_track] ) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) - result = geometry_dataset.intersecting_tracks(list(sections)) + result = geometry_dataset.intersecting_tracks(sections, BASE_GEOMETRY) assert result == {first_track.id, second_track.id} def test_as_dict(self, first_track: Track, second_track: Track) -> None: @@ -477,7 +482,7 @@ def test_contained_by_sections( ) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) result = geometry_dataset.contained_by_sections( - [not_intersecting_area_section, area_section] + [not_intersecting_area_section, area_section], BASE_GEOMETRY ) expected = { first_track.id: [ @@ -488,3 +493,37 @@ def test_contained_by_sections( ], } assert result == expected + + +class TestProfiling: + ROUNDS = 1 + ITERATIONS = 1 + WARMUP_ROUNDS = 0 + + @pytest.fixture + def tracks_15min(self, test_data_dir: Path) -> TrackDataset: + ottrk = test_data_dir / "OTCamera19_FR20_2023-05-24_07-00-00.ottrk" + ottrk_parser = OttrkParser(PandasDetectionParser(PandasByMaxConfidence())) + parse_result = ottrk_parser.parse(ottrk) + return parse_result.tracks + + @pytest.fixture + def sections(self, test_data_dir: Path) -> Iterable[Section]: + flow_file = test_data_dir / "OTCamera19_FR20_2023-05-24.otflow" + flow_parser = OtFlowParser() + sections, flows = flow_parser.parse(flow_file) + return sections + + def test_profile( + self, + benchmark: BenchmarkFixture, + tracks_15min: TrackDataset, + sections: Iterable[Section], + ) -> None: + benchmark.pedantic( + tracks_15min.intersecting_tracks, + args=(sections,), + rounds=self.ROUNDS, + iterations=self.ITERATIONS, + warmup_rounds=self.WARMUP_ROUNDS, + ) From 67537196e1139a6a4811f287ee65aa21905c9126 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:31:22 +0100 Subject: [PATCH 041/107] Change intersection strategies to use TrackDataset geometry operations --- OTAnalytics/application/analysis/intersect.py | 8 +- OTAnalytics/domain/intersect.py | 24 +- OTAnalytics/domain/track.py | 6 +- .../shapely/create_intersection_events.py | 288 ++++++++++-------- .../plugin_intersect/simple_intersect.py | 7 +- .../multiprocessing.py | 6 +- .../sequential.py | 10 +- .../test_create_intersection_events.py | 267 ++++++++++------ .../test_sequential.py | 23 +- tests/OTAnalytics/plugin_ui/test_cli.py | 15 +- 10 files changed, 400 insertions(+), 254 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index 61c9b69c7..f404d4108 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from collections import defaultdict -from typing import Iterable, Mapping, Sequence +from typing import Iterable, Mapping from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import RelativeOffsetCoordinate @@ -28,10 +28,10 @@ def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]] def group_sections_by_offset( - sections: Iterable[Section], -) -> Mapping[RelativeOffsetCoordinate, Sequence[Section]]: + sections: Iterable[Section], offset_type: EventType = EventType.SECTION_ENTER +) -> Mapping[RelativeOffsetCoordinate, list[Section]]: grouped_sections: dict[RelativeOffsetCoordinate, list[Section]] = defaultdict(list) for section in sections: - offset = section.get_offset(EventType.SECTION_ENTER) + offset = section.get_offset(offset_type) grouped_sections[offset].append(section) return grouped_sections diff --git a/OTAnalytics/domain/intersect.py b/OTAnalytics/domain/intersect.py index f4c07bc62..37d59d169 100644 --- a/OTAnalytics/domain/intersect.py +++ b/OTAnalytics/domain/intersect.py @@ -4,7 +4,7 @@ from OTAnalytics.domain.event import Event, EventBuilder from OTAnalytics.domain.geometry import Coordinate, Line, Polygon from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track import TrackDataset class IntersectImplementation(ABC): @@ -101,16 +101,17 @@ def num_processes(self) -> int: @abstractmethod def execute( self, - intersect: Callable[[Iterable[Track], Iterable[Section]], Iterable[Event]], - tasks: Sequence[tuple[Iterable[Track], Iterable[Section]]], + intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], ) -> list[Event]: """Executes the intersection of tracks with sections with the implemented parallelization strategy. Args: - intersect (Callable[[Track, Iterable[Section]], Iterable[Event]]): the - function to be executed on an iterable of tracks and sections. - tasks (tuple[Iterable[Track], Iterable[Section]) + intersect (Callable[[TrackDataset, Iterable[Section]], Iterable[Event]]): + the function to be executed on an iterable of tracks and sections. + tasks (Sequence[tuple[TrackDataset, Iterable[Section]]): the argument + to intersect function. Returns: Iterable[Event]: the generated events. @@ -135,17 +136,20 @@ class Intersector(ABC): @abstractmethod def intersect( - self, tracks: Iterable[Track], section: Section, event_builder: EventBuilder + self, + track_dataset: TrackDataset, + sections: Iterable[Section], + event_builder: EventBuilder, ) -> list[Event]: """Intersect tracks with sections and generate events if they intersect. Args: - tracks (Iterable[Track]): the tracks to be intersected with. - section (Section): the section to be intersected with. + track_dataset (TrackDataset): the tracks to be intersected with. + sections (Iterable[Section]): the sections to be intersected with. event_builder (EventBuilder): builder to generate events Returns: - list[Event]: the events if the track intersects with the section. + list[Event]: the events if the track intersects with the sections. Otherwise, return empty list. """ raise NotImplementedError diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 115e2bb05..233e8736a 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -186,13 +186,13 @@ def get_coordinate(self, offset: RelativeOffsetCoordinate | None) -> Coordinate: Returns: Coordinate: this detection's coordinate. """ - if offset: + if not offset or offset == RelativeOffsetCoordinate(0, 0): + return Coordinate(self.x, self.y) + else: return Coordinate( x=self.x + self.w * offset.x, y=self.y + self.h * offset.y, ) - else: - return Coordinate(self.x, self.y) class Track(ABC): diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index f78ed6a71..241a74320 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -1,16 +1,14 @@ from functools import singledispatchmethod from typing import Callable, Iterable -from shapely import LineString, Polygon, contains_xy, prepare +from shapely import LineString, Polygon from OTAnalytics.application.analysis.intersect import ( RunIntersect, group_sections_by_offset, ) from OTAnalytics.application.geometry import GeometryBuilder -from OTAnalytics.application.use_cases.track_repository import ( - GetTracksWithoutSingleDetections, -) +from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder from OTAnalytics.domain.geometry import ( DirectionVector2D, @@ -20,7 +18,7 @@ ) from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy from OTAnalytics.domain.section import Area, IntersectionVisitor, LineSection, Section -from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track import Track, TrackDataset, TrackId from OTAnalytics.domain.types import EventType @@ -85,6 +83,10 @@ def look_up(self, track: Track) -> LineString: return new_line +class IntersectionError(Exception): + pass + + class ShapelyIntersectBySmallestTrackSegments(Intersector): """ Implements the intersection strategy by splitting up the track in its smallest @@ -97,121 +99,168 @@ class ShapelyIntersectBySmallestTrackSegments(Intersector): def __init__( self, - geometry_builder: GeometryBuilder[LineString, Polygon], - track_lookup_table: ShapelyTrackLookupTable, calculate_direction_vector_: Callable[ [float, float, float, float], DirectionVector2D ] = calculate_direction_vector, ) -> None: - self._geometry_builder = geometry_builder self._calculate_direction_vector = calculate_direction_vector_ - self._track_table: ShapelyTrackLookupTable = track_lookup_table - self._segment_lookup_table: dict[TrackId, Iterable[LineString]] = {} - - def _lookup_segment(self, track: Track) -> Iterable[LineString]: - if segments := self._segment_lookup_table.get(track.id): - return segments - track_geometry = self._track_table.look_up(track) - new_segments = self._geometry_builder.create_line_segments(track_geometry) - self._segment_lookup_table[track.id] = new_segments - return new_segments def intersect( self, - tracks: Iterable[Track], - section: Section, + track_dataset: TrackDataset, + sections: Iterable[Section], event_builder: EventBuilder, ) -> list[Event]: + sections_grouped_by_offset = group_sections_by_offset( + sections, EventType.SECTION_ENTER + ) + events = [] + for offset, section_group in sections_grouped_by_offset.items(): + events.extend( + self.__do_intersect(track_dataset, section_group, offset, event_builder) + ) + return events + + def __do_intersect( + self, + track_dataset: TrackDataset, + sections: list[Section], + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder, + ) -> list[Event]: + intersection_result = track_dataset.intersection_points(sections, offset) + events: list[Event] = [] - section_geometry: LineString = self._geometry_builder.create_section(section) - for track in tracks: - track_geometry = self._track_table.look_up(track) - if not track_geometry.intersects(section_geometry): - continue - event_builder.add_road_user_type(track.classification) - - track_segments = self._lookup_segment(track) - for index, segment in enumerate(track_segments): - if segment.intersects(section_geometry): - x1, y1 = segment.coords[0] - x2, y2 = segment.coords[1] - direction_vector = self._calculate_direction_vector(x1, y1, x2, y2) - event_builder.add_event_type(EventType.SECTION_ENTER) - event_builder.add_direction_vector(direction_vector) - event_builder.add_event_coordinate(x2, y2) - events.append( - event_builder.create_event(track.detections[index + 1]) - ) + for track_id, intersection_points in intersection_result.items(): + if not (track := track_dataset.get_for(track_id)): + raise IntersectionError( + "Track not found. Unable to create intersection event " + f"for track {track_id}." + ) + for section_id, intersection_point in intersection_points: + event_builder.add_section_id(section_id) + event_builder.add_road_user_type(track.classification) + detection = track.detections[intersection_point.index] + current_coord = detection.get_coordinate(offset) + prev_coord = track.detections[ + intersection_point.index - 1 + ].get_coordinate(offset) + direction_vector = self._calculate_direction_vector( + prev_coord.x, + prev_coord.y, + current_coord.x, + current_coord.y, + ) + event_builder.add_event_type(EventType.SECTION_ENTER) + event_builder.add_direction_vector(direction_vector) + event_builder.add_event_coordinate(detection.x, detection.y) + events.append(event_builder.create_event(detection)) return events class ShapelyIntersectAreaByTrackPoints(Intersector): def __init__( self, - geometry_builder: GeometryBuilder[LineString, Polygon], - track_lookup_table: ShapelyTrackLookupTable, calculate_direction_vector_: Callable[ [float, float, float, float], DirectionVector2D ] = calculate_direction_vector, ) -> None: - self._geometry_builder = geometry_builder self._calculate_direction_vector = calculate_direction_vector_ - self._track_table = track_lookup_table def intersect( - self, tracks: Iterable[Track], section: Section, event_builder: EventBuilder + self, + track_dataset: TrackDataset, + sections: Iterable[Section], + event_builder: EventBuilder, ) -> list[Event]: + sections_grouped_by_offset = group_sections_by_offset( + sections, EventType.SECTION_ENTER + ) events = [] + for offset, section_group in sections_grouped_by_offset.items(): + events.extend( + self.__do_intersect(track_dataset, section_group, offset, event_builder) + ) + return events - section_geometry = self._geometry_builder.create_section(section) - prepare(section_geometry) + def __do_intersect( + self, + track_dataset: TrackDataset, + sections: list[Section], + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder, + ) -> list[Event]: + contained_by_sections_result = track_dataset.contained_by_sections( + sections, offset + ) - for track in tracks: - track_coordinates = self._track_table.look_up(track).coords - section_entered_mask = contains_xy(section_geometry, track_coordinates) + events = [] + for ( + track_id, + contained_by_sections_masks, + ) in contained_by_sections_result.items(): + if not (track := track_dataset.get_for(track_id)): + raise IntersectionError( + "Track not found. Unable to create intersection event " + f"for track {track_id}." + ) + track_detections = track.detections + for section_id, section_entered_mask in contained_by_sections_masks: + event_builder.add_section_id(section_id) + event_builder.add_road_user_type(track.classification) - track_starts_inside_area = section_entered_mask[0] - event_builder.add_road_user_type(track.classification) - if track_starts_inside_area: - first_detection = track.first_detection - first_coord_x, first_coord_y = track_coordinates[0] - second_coord_x, second_coord_y = track_coordinates[1] - event_builder.add_event_type(EventType.SECTION_ENTER) - event_builder.add_direction_vector( - self._calculate_direction_vector( - first_coord_x, first_coord_y, second_coord_x, second_coord_y + track_starts_inside_area = section_entered_mask[0] + if track_starts_inside_area: + first_detection = track_detections[0] + first_coord = first_detection.get_coordinate(offset) + second_coord = track_detections[1].get_coordinate(offset) + + event_builder.add_event_type(EventType.SECTION_ENTER) + event_builder.add_direction_vector( + self._calculate_direction_vector( + first_coord.x, + first_coord.y, + second_coord.x, + second_coord.y, + ) ) - ) - event_builder.add_event_coordinate(first_coord_x, first_coord_y) - event = event_builder.create_event(first_detection) - events.append(event) - - section_currently_entered = track_starts_inside_area - - for current_index, current_detection in enumerate( - track.detections[1:], start=1 - ): - entered = section_entered_mask[current_index] - if section_currently_entered == entered: - continue - current_x, current_y = track_coordinates[current_index] - prev_x, prev_y = track_coordinates[current_index - 1] - - event_builder.add_direction_vector( - self._calculate_direction_vector( - prev_x, prev_y, current_x, current_y + event_builder.add_event_coordinate( + first_detection.x, first_detection.y ) - ) - event_builder.add_event_coordinate(current_x, current_y) + event = event_builder.create_event(first_detection) + events.append(event) + + section_currently_entered = track_starts_inside_area + for current_index, current_detection in enumerate( + track_detections[1:], start=1 + ): + entered = section_entered_mask[current_index] + if section_currently_entered == entered: + continue + + prev_coord = track_detections[current_index - 1].get_coordinate( + offset + ) + current_coord = current_detection.get_coordinate(offset) + + event_builder.add_direction_vector( + self._calculate_direction_vector( + prev_coord.x, + prev_coord.y, + current_coord.x, + current_coord.y, + ) + ) + event_builder.add_event_coordinate(current_coord.x, current_coord.y) - if entered: - event_builder.add_event_type(EventType.SECTION_ENTER) - else: - event_builder.add_event_type(EventType.SECTION_LEAVE) + if entered: + event_builder.add_event_type(EventType.SECTION_ENTER) + else: + event_builder.add_event_type(EventType.SECTION_LEAVE) - event = event_builder.create_event(current_detection) - events.append(event) - section_currently_entered = entered + event = event_builder.create_event(current_detection) + events.append(event) + section_currently_entered = entered return events @@ -221,43 +270,43 @@ def __init__( self, intersect_line_section: Intersector, intersect_area_section: Intersector, - geometry_builder: GeometryBuilder, - tracks: Iterable[Track], + track_dataset: TrackDataset, sections: Iterable[Section], event_builder: SectionEventBuilder, ): self._intersect_line_section = intersect_line_section self._intersect_area_section = intersect_area_section - self._geometry_builder = geometry_builder - self._tracks = tracks + self._track_dataset = track_dataset self._sections = sections self._event_builder = event_builder def create(self) -> list[Event]: events = [] - - for section in self._sections: - events.extend(section.accept(self)) + line_sections, area_sections = separate_sections(self._sections) + events.extend( + self._intersect_line_section.intersect( + self._track_dataset, line_sections, self._event_builder + ) + ) + events.extend( + self._intersect_area_section.intersect( + self._track_dataset, area_sections, self._event_builder + ) + ) return events def intersect_line_section(self, section: LineSection) -> list[Event]: - self._event_builder.add_section_id(section.id) - return self._intersect_line_section.intersect( - self._tracks, section, self._event_builder - ) + raise NotImplementedError def intersect_area_section(self, section: Area) -> list[Event]: - self._event_builder.add_section_id(section.id) - return self._intersect_area_section.intersect( - self._tracks, section, self._event_builder - ) + raise NotImplementedError class ShapelyRunIntersect(RunIntersect): def __init__( self, intersect_parallelizer: IntersectParallelizationStrategy, - get_tracks: GetTracksWithoutSingleDetections, + get_tracks: GetAllTracks, ) -> None: self._intersect_parallelizer = intersect_parallelizer self._get_tracks = get_tracks @@ -271,29 +320,18 @@ def __call__(self, sections: Iterable[Section]) -> list[Event]: return self._intersect_parallelizer.execute(_create_events, tasks) -def _create_events(tracks: Iterable[Track], sections: Iterable[Section]) -> list[Event]: - grouped_sections = group_sections_by_offset(sections) +def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Event]: events = [] - for offset, section_group in grouped_sections.items(): - geometry_builder = ShapelyGeometryBuilder() - track_lookup_table = ShapelyTrackLookupTable(dict(), geometry_builder, offset) - line_section_intersection_strategy = ShapelyIntersectBySmallestTrackSegments( - geometry_builder, track_lookup_table - ) - area_section_intersection_strategy = ShapelyIntersectAreaByTrackPoints( - geometry_builder, track_lookup_table - ) - event_builder = SectionEventBuilder() - - create_intersection_events = ShapelyCreateIntersectionEvents( - intersect_line_section=line_section_intersection_strategy, - intersect_area_section=area_section_intersection_strategy, - geometry_builder=geometry_builder, - tracks=tracks, - sections=section_group, - event_builder=event_builder, - ) - events.extend(create_intersection_events.create()) + event_builder = SectionEventBuilder() + + create_intersection_events = ShapelyCreateIntersectionEvents( + intersect_line_section=ShapelyIntersectBySmallestTrackSegments(), + intersect_area_section=ShapelyIntersectAreaByTrackPoints(), + track_dataset=tracks, + sections=sections, + event_builder=event_builder, + ) + events.extend(create_intersection_events.create()) return events diff --git a/OTAnalytics/plugin_intersect/simple_intersect.py b/OTAnalytics/plugin_intersect/simple_intersect.py index aaa460a38..d5cb8ad2b 100644 --- a/OTAnalytics/plugin_intersect/simple_intersect.py +++ b/OTAnalytics/plugin_intersect/simple_intersect.py @@ -5,6 +5,7 @@ from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.types import EventType class SimpleTracksIntersectingSections(TracksIntersectingSections): @@ -20,7 +21,11 @@ def intersect(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId] total_tracks_intersected = 0 print("Number of intersecting tracks per section") for section in sections: - result[section.id].update(track_dataset.intersecting_tracks([section])) + result[section.id].update( + track_dataset.intersecting_tracks( + [section], section.get_offset(EventType.SECTION_ENTER) + ) + ) num_tracks_intersected = len(result[section.id]) total_tracks_intersected += num_tracks_intersected print(f"{section.name}: {num_tracks_intersected} tracks") diff --git a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py index d06a577c9..2377d95f2 100644 --- a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py +++ b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py @@ -7,7 +7,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track import TrackDataset class MultiprocessingIntersectParallelization(IntersectParallelizationStrategy): @@ -31,8 +31,8 @@ def set_num_processes(self, value: int) -> None: def execute( self, - intersect: Callable[[Iterable[Track], Iterable[Section]], Iterable[Event]], - tasks: Sequence[tuple[Iterable[Track], Iterable[Section]]], + intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], ) -> list[Event]: logger().debug( f"Start intersection in parallel with {self._num_processes} processes." diff --git a/OTAnalytics/plugin_intersect_parallelization/sequential.py b/OTAnalytics/plugin_intersect_parallelization/sequential.py index e6315dbc4..6c8d9aa9b 100644 --- a/OTAnalytics/plugin_intersect_parallelization/sequential.py +++ b/OTAnalytics/plugin_intersect_parallelization/sequential.py @@ -3,7 +3,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track import TrackDataset class SequentialIntersect(IntersectParallelizationStrategy): @@ -15,13 +15,13 @@ def num_processes(self) -> int: def execute( self, - intersect: Callable[[Iterable[Track], Iterable[Section]], Iterable[Event]], - tasks: Sequence[tuple[Iterable[Track], Iterable[Section]]], + intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], ) -> list[Event]: events: list[Event] = [] for task in tasks: - tracks, sections = task - events.extend(intersect(tracks, sections)) + track_dataset, sections = task + events.extend(intersect(track_dataset, sections)) return events def set_num_processes(self, value: int) -> None: diff --git a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py index e6575dfc5..3a69cafad 100644 --- a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py +++ b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py @@ -16,13 +16,11 @@ SectionId, SectionType, ) -from OTAnalytics.domain.track import Detection, Track +from OTAnalytics.domain.track import Detection, IntersectionPoint, Track, TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( - ShapelyGeometryBuilder, ShapelyIntersectAreaByTrackPoints, ShapelyIntersectBySmallestTrackSegments, - ShapelyTrackLookupTable, separate_sections, ) from tests.conftest import TrackBuilder @@ -52,12 +50,20 @@ def from_detection( return _ExpectedEventCoord(detection_index, x, y, event_type) -@dataclass class _TestCase: - track: Track - section: Section - expected_event_coords: list[_ExpectedEventCoord] - direction_vectors: list[tuple[float, float]] + def __init__( + self, + track: Track, + track_dataset: Mock, + section: Section, + expected_event_coords: list[_ExpectedEventCoord], + direction_vectors: list[tuple[float, float]], + ): + self.track = track + self.track_dataset = track_dataset + self.section = section + self.expected_event_coords = expected_event_coords + self.direction_vectors = direction_vectors def assert_valid(self, event_results: list, event_builder: Mock) -> None: assert len(event_results) == len(self.expected_event_coords) @@ -94,6 +100,52 @@ def _assert_no_calls(self, event_builder: Mock) -> None: event_builder.add_event.assert_not_called() +class LineSectionTestCase(_TestCase): + def __init__( + self, + track: Track, + track_dataset: Mock, + section: Section, + expected_event_coords: list[_ExpectedEventCoord], + direction_vectors: list[tuple[float, float]], + ): + super().__init__( + track, track_dataset, section, expected_event_coords, direction_vectors + ) + + def assert_valid(self, event_results: list, event_builder: Mock) -> None: + super().assert_valid(event_results, event_builder) + self._assert_valid() + + def _assert_valid(self) -> None: + self.track_dataset.intersection_points.assert_called_once_with( + [self.section], self.section.get_offset(EventType.SECTION_ENTER) + ) + + +class AreaSectionTestCase(_TestCase): + def __init__( + self, + track: Track, + track_dataset: Mock, + section: Section, + expected_event_coords: list[_ExpectedEventCoord], + direction_vectors: list[tuple[float, float]], + ): + super().__init__( + track, track_dataset, section, expected_event_coords, direction_vectors + ) + + def assert_valid(self, event_results: list, event_builder: Mock) -> None: + super().assert_valid(event_results, event_builder) + self._assert_valid() + + def _assert_valid(self) -> None: + self.track_dataset.contained_by_sections.assert_called_once_with( + [self.section], self.section.get_offset(EventType.SECTION_ENTER) + ) + + def create_section( coordinates: list[tuple[float, float]], section_type: SectionType, @@ -167,39 +219,42 @@ def closed_track(track_builder: TrackBuilder) -> Track: track_builder.add_frame(1) track_builder.add_second(1) - track_builder.add_xy_bbox(1, 1) + track_builder.add_xy_bbox(1.0, 1.0) track_builder.append_detection() track_builder.add_frame(2) track_builder.add_second(2) - track_builder.add_xy_bbox(2, 1) + track_builder.add_xy_bbox(2.0, 1.0) track_builder.append_detection() track_builder.add_frame(3) track_builder.add_second(3) - track_builder.add_xy_bbox(2, 2) + track_builder.add_xy_bbox(2.0, 2.0) track_builder.append_detection() track_builder.add_frame(5) track_builder.add_second(5) - track_builder.add_xy_bbox(1, 2) + track_builder.add_xy_bbox(1.0, 2.0) track_builder.append_detection() track_builder.add_frame(5) track_builder.add_second(5) - track_builder.add_xy_bbox(1, 1) + track_builder.add_xy_bbox(1.0, 1.0) track_builder.append_detection() return track_builder.build_track() @pytest.fixture -def test_case_track_line_section( - track: Track, -) -> _TestCase: +def test_case_track_line_section(track: Track) -> _TestCase: offset = (0, 0.5) section = create_section([(1.5, 0), (1.5, 1.5)], SectionType.LINE, offset) detection_index = 2 detection = track.detections[detection_index] + track_dataset = Mock(spec=TrackDataset) + track_dataset.intersection_points.return_value = { + track.id: [(section.id, IntersectionPoint(detection_index))] + } + track_dataset.get_for.return_value = track expected_event_coords = [ _ExpectedEventCoord.from_detection( detection, @@ -210,7 +265,9 @@ def test_case_track_line_section( ] direction_vectors = [(1.0, 0.0)] - return _TestCase(track, section, expected_event_coords, direction_vectors) + return LineSectionTestCase( + track, track_dataset, section, expected_event_coords, direction_vectors + ) @pytest.fixture @@ -218,31 +275,41 @@ def test_case_closed_track_line_section( closed_track: Track, ) -> _TestCase: section = create_section([(0, 1.5), (3, 1.5)], SectionType.LINE) + track_dataset = Mock(spec=TrackDataset) + track_dataset.intersection_points.return_value = { + closed_track.id: [ + (section.id, IntersectionPoint(2)), + (section.id, IntersectionPoint(4)), + ] + } + track_dataset.get_for.return_value = track expected_event_coords = [ _ExpectedEventCoord(2, 2.0, 2.0), _ExpectedEventCoord(4, 1.0, 1.0), ] expected_dir_vectors = [(0.0, 1.0), (0.0, -1.0)] - return _TestCase(closed_track, section, expected_event_coords, expected_dir_vectors) + return _TestCase( + closed_track, + track_dataset, + section, + expected_event_coords, + expected_dir_vectors, + ) @pytest.fixture def test_case_line_section_no_intersection(track: Track) -> _TestCase: section = create_section([(0, 0), (10, 0)], SectionType.LINE) + track_dataset = Mock(spec=TrackDataset) + track_dataset.intersection_points.return_value = {} + track_dataset.get_for.return_value = track - return _TestCase(track, section, [], []) + return _TestCase(track, track_dataset, section, [], []) class TestShapelyIntersectBySmallestTrackSegments: - def _create_intersector( - self, offset: RelativeOffsetCoordinate - ) -> ShapelyIntersectBySmallestTrackSegments: - geometry_builder = ShapelyGeometryBuilder() - track_lookup_table = ShapelyTrackLookupTable(dict(), geometry_builder, offset) - - return ShapelyIntersectBySmallestTrackSegments( - geometry_builder, track_lookup_table - ) + def _create_intersector(self) -> ShapelyIntersectBySmallestTrackSegments: + return ShapelyIntersectBySmallestTrackSegments() @pytest.mark.parametrize( "test_case_name", @@ -264,24 +331,17 @@ def test_intersect( event_builder = Mock() event_builder.create_event.return_value = event - intersector = self._create_intersector( - test_case.section.get_offset(EventType.SECTION_ENTER) - ) + intersector = self._create_intersector() result_events = intersector.intersect( - [test_case.track], test_case.section, event_builder + test_case.track_dataset, [test_case.section], event_builder ) test_case.assert_valid(result_events, event_builder) class TestShapelyIntersectAreaByTrackPoints: - def _create_intersector( - self, offset: RelativeOffsetCoordinate - ) -> ShapelyIntersectAreaByTrackPoints: - geometry_builder = ShapelyGeometryBuilder() - track_lookup_table = ShapelyTrackLookupTable(dict(), geometry_builder, offset) - - return ShapelyIntersectAreaByTrackPoints(geometry_builder, track_lookup_table) + def _create_intersector(self) -> ShapelyIntersectAreaByTrackPoints: + return ShapelyIntersectAreaByTrackPoints() @pytest.fixture def straight_track(self, track_builder: TrackBuilder) -> Track: @@ -343,6 +403,11 @@ def test_case_track_starts_outside_section( SectionType.AREA, offset, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + straight_track.id: [(section.id, [False, True, False])] + } + track_dataset.get_for.return_value = straight_track expected_event_coords = [ _ExpectedEventCoord.from_detection( straight_track.detections[1], 1, EventType.SECTION_ENTER, offset @@ -352,8 +417,12 @@ def test_case_track_starts_outside_section( ), ] expected_direction_vectors = [(1.0, 0.0), (1.0, 0)] - return _TestCase( - straight_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + straight_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture @@ -362,13 +431,22 @@ def test_case_track_starts_inside_section(self, straight_track: Track) -> _TestC [(0.5, 0.5), (0.5, 1.5), (1.5, 1.5), (1.5, 0.5), (0.5, 0.5)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + straight_track.id: [(section.id, [True, False, False])] + } + track_dataset.get_for.return_value = straight_track expected_event_coords = [ _ExpectedEventCoord(0, 1.0, 1.0, EventType.SECTION_ENTER), _ExpectedEventCoord(1, 2.0, 1.0, EventType.SECTION_LEAVE), ] expected_direction_vectors = [(1.0, 0.0), (1.0, 0.0)] - return _TestCase( - straight_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + straight_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture @@ -377,12 +455,21 @@ def test_case_track_is_inside_section(self, straight_track: Track) -> _TestCase: [(0.0, 0.0), (0.0, 2.0), (4.0, 2.0), (4.0, 0.0), (0.0, 0.0)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + straight_track.id: [(section.id, [True, True, True])] + } + track_dataset.get_for.return_value = straight_track expected_event_coords = [ _ExpectedEventCoord(0, 1.0, 1.0, EventType.SECTION_ENTER) ] expected_direction_vectors = [(1.0, 0.0)] - return _TestCase( - straight_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + straight_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture @@ -391,12 +478,21 @@ def test_case_starts_outside_stays_inside(self, straight_track: Track) -> _TestC [(1.5, 0.5), (1.5, 1.5), (4.0, 1.5), (4.0, 0.5), (1.5, 0.5)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + straight_track.id: [(section.id, [False, True, True])] + } + track_dataset.get_for.return_value = straight_track expected_event_coords = [ _ExpectedEventCoord(1, 2.0, 1.0, EventType.SECTION_ENTER) ] expected_direction_vectors = [(1.0, 0.0)] - return _TestCase( - straight_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + straight_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture @@ -408,14 +504,23 @@ def test_case_track_starts_outside_section_multiple_intersections( [(1.5, 0.5), (1.5, 2.5), (2.5, 2.5), (2.5, 0.5), (1.5, 0.5)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + complex_track.id: [(section.id, [False, True, True, False, False, True])] + } + track_dataset.get_for.return_value = complex_track expected_event_coords = [ _ExpectedEventCoord(1, 2.0, 1.0, EventType.SECTION_ENTER), _ExpectedEventCoord(3, 1.0, 1.5, EventType.SECTION_LEAVE), _ExpectedEventCoord(5, 2.0, 2.0, EventType.SECTION_ENTER), ] expected_direction_vectors = [(1.0, 0.0), (-1.0, 0), (1.0, 0.0)] - return _TestCase( - complex_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + complex_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture @@ -427,6 +532,11 @@ def test_case_track_starts_inside_section_multiple_intersections( [(0.5, 0.5), (0.5, 2.5), (1.5, 2.5), (1.5, 0.5), (0.5, 0.5)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + complex_track.id: [(section.id, [True, False, False, True, True, False])] + } + track_dataset.get_for.return_value = complex_track expected_event_coords = [ _ExpectedEventCoord(0, 1.0, 1.0, EventType.SECTION_ENTER), _ExpectedEventCoord(1, 2.0, 1.0, EventType.SECTION_LEAVE), @@ -434,49 +544,36 @@ def test_case_track_starts_inside_section_multiple_intersections( _ExpectedEventCoord(5, 2.0, 2.0, EventType.SECTION_LEAVE), ] expected_direction_vectors = [(1.0, 0.0), (1.0, 0.0), (-1.0, 0), (1, 0)] - return _TestCase( - complex_track, section, expected_event_coords, expected_direction_vectors + return AreaSectionTestCase( + complex_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.fixture - def test_case_intersect_closed_track( - self, track_builder: TrackBuilder - ) -> _TestCase: - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 1.0) - track_builder.add_frame(2) - track_builder.add_microsecond(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 1.5) - track_builder.add_frame(3) - track_builder.add_microsecond(2) - track_builder.append_detection() - - track_builder.add_xy_bbox(1.0, 1.5) - track_builder.add_frame(4) - track_builder.add_microsecond(3) - track_builder.append_detection() - - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.add_frame(5) - track_builder.add_microsecond(4) - track_builder.append_detection() - - track = track_builder.build_track() + def test_case_intersect_closed_track(self, closed_track: Track) -> _TestCase: section = create_section( [(1.5, 0.5), (1.5, 2.0), (2.5, 2.0), (2.5, 0.5), (1.5, 0.5)], SectionType.AREA, ) + track_dataset = Mock(spec=TrackDataset) + track_dataset.contained_by_sections.return_value = { + closed_track.id: [(section.id, [False, True, False, False, False])] + } + track_dataset.get_for.return_value = closed_track expected_event_coords = [ _ExpectedEventCoord(1, 2.0, 1.0, EventType.SECTION_ENTER), - _ExpectedEventCoord(3, 1.0, 1.5, EventType.SECTION_LEAVE), + _ExpectedEventCoord(2, 2.0, 2.0, EventType.SECTION_LEAVE), ] - expected_direction_vectors = [(1.0, 0.0), (-1.0, 0.0)] - return _TestCase( - track, section, expected_event_coords, expected_direction_vectors + expected_direction_vectors = [(1.0, 0.0), (0.0, 1.0)] + return AreaSectionTestCase( + closed_track, + track_dataset, + section, + expected_event_coords, + expected_direction_vectors, ) @pytest.mark.parametrize( @@ -503,11 +600,9 @@ def test_intersect( event_builder = Mock() event_builder.create_event.return_value = event - intersector = self._create_intersector( - test_case.section.get_offset(EventType.SECTION_ENTER) - ) + intersector = self._create_intersector() result_events = intersector.intersect( - [test_case.track], test_case.section, event_builder + test_case.track_dataset, [test_case.section], event_builder ) test_case.assert_valid(result_events, event_builder) diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py index f54c51c93..500b9d708 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py @@ -1,6 +1,9 @@ +from typing import Callable, cast from unittest.mock import Mock, call from OTAnalytics.domain.event import Event +from OTAnalytics.domain.section import Section +from OTAnalytics.domain.track import TrackDataset from OTAnalytics.plugin_intersect_parallelization.sequential import SequentialIntersect @@ -10,20 +13,20 @@ def test_execute(self) -> None: event_2 = Mock(spec=Event) side_effect = [[event_1], [event_2]] - mock_intersect = Mock(spec=callable, side_effect=side_effect) - first_track = Mock() - second_track = Mock() - sections = [Mock()] + mock_intersect = Mock(spec=Callable, side_effect=side_effect) + first_track_dataset = Mock() + second_track_dataset = Mock() + sections: list[Section] = [Mock()] - tasks = [ - ([first_track], sections), - ([second_track], sections), + tasks: list[tuple[TrackDataset, list[Section]]] = [ + (first_track_dataset, sections), + (second_track_dataset, sections), ] sequential_intersect = SequentialIntersect() - result = sequential_intersect.execute(mock_intersect, tasks) + result = sequential_intersect.execute(cast(Callable, mock_intersect), tasks) assert result == [event_1, event_2] assert mock_intersect.call_args_list == [ - call([first_track], sections), - call([second_track], sections), + call(first_track_dataset, sections), + call(second_track_dataset, sections), ] diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 7d4473988..5f5c75677 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -44,6 +44,7 @@ AddAllTracks, ClearAllTracks, GetAllTrackIds, + GetAllTracks, GetTracksFromIds, GetTracksWithoutSingleDetections, RemoveTracks, @@ -59,7 +60,6 @@ from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( ShapelyRunIntersect, ) -from OTAnalytics.plugin_intersect.shapely.intersect import ShapelyIntersector from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTrackSegmentBuilder, @@ -267,7 +267,10 @@ def cli_dependencies(self) -> dict[str, Any]: flow_repository = FlowRepository() add_events = AddEvents(event_repository) - get_all_tracks = GetTracksWithoutSingleDetections(track_repository) + get_tracks_without_single_detections = GetTracksWithoutSingleDetections( + track_repository + ) + get_all_tracks = GetAllTracks(track_repository) get_all_track_ids = GetAllTrackIds(track_repository) add_all_tracks = AddAllTracks(track_repository) clear_all_tracks = ClearAllTracks(track_repository) @@ -281,9 +284,7 @@ def cli_dependencies(self) -> dict[str, Any]: section_repository, add_events, ) - tracks_intersecting_sections = SimpleTracksIntersectingSections( - get_all_tracks, ShapelyIntersector() - ) + tracks_intersecting_sections = SimpleTracksIntersectingSections(get_all_tracks) cut_tracks_with_section = SimpleCutTracksWithSection( GetTracksFromIds(track_repository), ShapelyMapper(), @@ -293,7 +294,7 @@ def cli_dependencies(self) -> dict[str, Any]: cut_tracks = ( SimpleCutTracksIntersectingSection( GetSectionsById(section_repository), - get_all_tracks, + get_tracks_without_single_detections, tracks_intersecting_sections, cut_tracks_with_section, add_all_tracks, @@ -302,7 +303,7 @@ def cli_dependencies(self) -> dict[str, Any]: ), ) create_scene_events = SimpleCreateSceneEvents( - get_all_tracks, + get_tracks_without_single_detections, SceneActionDetector(SceneEventBuilder()), add_events, ) From e0f5c52aa31e597ddbb161670bccfda9e2a36fc8 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:08:26 +0100 Subject: [PATCH 042/107] Make track_ids property part of TrackGeometryDataset interface --- OTAnalytics/domain/track.py | 10 ++++++++++ .../track_geometry_store/pygeos_store.py | 5 ----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 233e8736a..1455901ec 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -689,6 +689,16 @@ class TrackGeometryDataset(ABC): coordinates. """ + @property + @abstractmethod + def track_ids(self) -> set[str]: + """Get track ids of tracks stored in dataset. + + Returns: + set[str]: the track ids stored. + """ + raise NotImplementedError + @staticmethod @abstractmethod def from_track_dataset(dataset: TrackDataset) -> "TrackGeometryDataset": diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 40853b55f..c9b0ef713 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -120,11 +120,6 @@ def _check_is_valid( @property def track_ids(self) -> set[str]: - """Get track ids of tracks stored in dataset. - - Returns: - set[str]: the track ids stored. - """ return set(self._dataset[BASE_GEOMETRY].index) @property From 19578b09f0098e27daeb9c67296d1bc956c703a4 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:10:33 +0100 Subject: [PATCH 043/107] Reuse existing geometry dataset when adding new tracks --- OTAnalytics/plugin_datastore/track_store.py | 43 ++++++++++------ OTAnalytics/plugin_parser/pandas_parser.py | 4 +- .../plugin_datastore/test_track_store.py | 49 +++++++++++++------ 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 63cd39609..fab77440c 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -18,6 +18,7 @@ IntersectionPoint, Track, TrackDataset, + TrackGeometryDataset, TrackId, ) from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( @@ -149,6 +150,7 @@ class PandasTrackDataset(TrackDataset): def __init__( self, dataset: DataFrame = DataFrame(), + geometry_dataset: TrackGeometryDataset | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( PygeosTrackGeometryDataset.from_track_dataset @@ -158,18 +160,24 @@ def __init__( self._calculator = calculator self._track_geometry_factory = track_geometry_factory # TODO: Re-use existing track geometries instead of creating new ones - self._track_geometry_dataset = self._track_geometry_factory(self) + if geometry_dataset is None: + self._geometry_dataset = self._track_geometry_factory(self) + else: + self._geometry_dataset = geometry_dataset @staticmethod def from_list( tracks: list[Track], calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ) -> TrackDataset: - return PandasTrackDataset.from_dataframe(_convert_tracks(tracks), calculator) + return PandasTrackDataset.from_dataframe( + _convert_tracks(tracks), calculator=calculator + ) @staticmethod def from_dataframe( tracks: DataFrame, + track_geometry_dataset: TrackGeometryDataset | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ) -> TrackDataset: if tracks.empty: @@ -187,16 +195,23 @@ def from_dataframe( valid_tracks = classified_tracks.loc[ classified_tracks[track.TRACK_ID].isin(valid_track_ids), : ] - return PandasTrackDataset(valid_tracks) + return PandasTrackDataset(valid_tracks, geometry_dataset=track_geometry_dataset) def add_all(self, other: Iterable[Track]) -> TrackDataset: new_tracks = self.__get_tracks(other) if new_tracks.empty: return self if self._dataset.empty: - return PandasTrackDataset.from_dataframe(new_tracks, self._calculator) + return PandasTrackDataset.from_dataframe( + new_tracks, calculator=self._calculator + ) new_dataset = pandas.concat([self._dataset, new_tracks]) - return PandasTrackDataset.from_dataframe(new_dataset) + new_track_ids = new_tracks[track.TRACK_ID].unique() + new_tracks_df = new_dataset[new_dataset[track.TRACK_ID].isin(new_track_ids)] + updated_geometry_dataset = self._geometry_dataset.add_all( + PandasTrackDataset.from_dataframe(new_tracks_df) + ) + return PandasTrackDataset.from_dataframe(new_dataset, updated_geometry_dataset) def get_all_ids(self) -> Iterable[TrackId]: return self._dataset[track.TRACK_ID].apply(lambda track_id: TrackId(track_id)) @@ -241,8 +256,9 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: new_batches: list["TrackDataset"] = [] for batch_ids in batched(all_ids, batch_size): batch_dataset = self._dataset[self._dataset[track.TRACK_ID].isin(batch_ids)] - new_batches.append(PandasTrackDataset(batch_dataset, self._calculator)) - + new_batches.append( + PandasTrackDataset(batch_dataset, calculator=self._calculator) + ) return new_batches def __len__(self) -> int: @@ -261,22 +277,22 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": filtered_dataset = self._dataset.loc[ self._dataset[track.TRACK_ID].isin(filtered_ids) ] - return PandasTrackDataset(filtered_dataset, self._calculator) + return PandasTrackDataset(filtered_dataset, calculator=self._calculator) def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> set[TrackId]: - return self._track_geometry_dataset.intersecting_tracks(sections, offset) + return self._geometry_dataset.intersecting_tracks(sections, offset) def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._track_geometry_dataset.intersection_points(sections, offset) + return self._geometry_dataset.intersection_points(sections, offset) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - return self._track_geometry_dataset.contained_by_sections(sections, offset) + return self._geometry_dataset.contained_by_sections(sections, offset) def _assign_track_classification( @@ -322,6 +338,5 @@ def _sort_tracks(track_df: DataFrame) -> DataFrame: DataFrame: sorted dataframe by track id and frame """ if (track.TRACK_ID in track_df.columns) and (track.FRAME in track_df.columns): - return track_df.sort_values([track.TRACK_ID, track.FRAME]) - else: - return track_df + track_df.sort_values([track.FRAME], inplace=True) + return track_df diff --git a/OTAnalytics/plugin_parser/pandas_parser.py b/OTAnalytics/plugin_parser/pandas_parser.py index 6ca968f57..47f5cbc1d 100644 --- a/OTAnalytics/plugin_parser/pandas_parser.py +++ b/OTAnalytics/plugin_parser/pandas_parser.py @@ -83,4 +83,6 @@ def _parse_as_dataframe( tracks_to_remain.sort_values( by=[track.TRACK_ID, track.OCCURRENCE], inplace=True ) - return PandasTrackDataset.from_dataframe(tracks_to_remain, self._calculator) + return PandasTrackDataset.from_dataframe( + tracks_to_remain, calculator=self._calculator + ) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index ce4b340d4..4f1e5549b 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -1,3 +1,5 @@ +from typing import cast + import pytest from pandas import DataFrame, Series @@ -91,7 +93,6 @@ def test_add(self) -> None: assert 0 == len(dataset.as_list()) for actual, expected in zip(merged, expected_dataset): assert_equal_track_properties(actual, expected) - # assert merged == expected_dataset def test_add_nothing(self) -> None: dataset = PandasTrackDataset() @@ -103,18 +104,30 @@ def test_add_nothing(self) -> None: def test_add_all(self) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - expected_dataset = PandasTrackDataset.from_list([first_track, second_track]) - dataset = PandasTrackDataset() - merged = dataset.add_all( - PythonTrackDataset( - {first_track.id: first_track, second_track.id: second_track} - ) + third_track = self.__build_track("3") + expected_dataset = PandasTrackDataset.from_list( + [first_track, second_track, third_track] + ) + dataset = PandasTrackDataset.from_list([]) + assert len(dataset) == 0 + dataset = dataset.add_all([first_track]) + assert len(dataset) == 1 + merged = cast( + PandasTrackDataset, + dataset.add_all( + PythonTrackDataset( + {second_track.id: second_track, third_track.id: third_track} + ) + ), ) - - assert merged == expected_dataset for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) - assert 0 == len(dataset.as_list()) + assert merged._geometry_dataset.track_ids == { + first_track.id.id, + second_track.id.id, + third_track.id.id, + } + assert 1 == len(dataset.as_list()) def test_add_two_existing_pandas_datasets(self) -> None: first_track = self.__build_track("1") @@ -122,12 +135,17 @@ def test_add_two_existing_pandas_datasets(self) -> None: expected_dataset = PandasTrackDataset.from_list([first_track, second_track]) first = PandasTrackDataset.from_list([first_track]) second = PandasTrackDataset.from_list([second_track]) - merged = first.add_all(second) + merged = cast(PandasTrackDataset, first.add_all(second)) - assert merged == expected_dataset for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) + # TODO: Is it enough to check for track ids in the geometry dataset? + assert merged._geometry_dataset.track_ids == { + first_track.id.id, + second_track.id.id, + } + def __build_track(self, track_id: str, length: int = 5) -> Track: builder = TrackBuilder() builder.add_track_id(track_id) @@ -167,8 +185,11 @@ def test_remove(self) -> None: dataset = PandasTrackDataset.from_list([first_track, second_track]) removed_track_set = dataset.remove(first_track.id) - - assert PandasTrackDataset.from_list([second_track]) == removed_track_set + for actual, expected in zip( + removed_track_set.as_list(), + PandasTrackDataset.from_list([second_track]).as_list(), + ): + assert_equal_track_properties(actual, expected) def test_len(self) -> None: first_track = self.__build_track("1") From 08c4e79715843225a6f4819aa1114124da3b82e0 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:11:53 +0100 Subject: [PATCH 044/107] Move responsibility to manage track geometries for different offsets from TrackGeometryDataset to TrackDataset The TrackDataset now manages geometries for different offsets. A TrackGeometryDataset now only maintains a DataFrame of track geometries. A TrackGeometryDataset holds geometries for one offset. Adding new tracks to it will implicitly generate geometries with new offset applied. --- OTAnalytics/domain/track.py | 19 ++- .../plugin_datastore/python_track_store.py | 48 ++++++- .../track_geometry_store/pygeos_store.py | 109 +++++++--------- OTAnalytics/plugin_datastore/track_store.py | 53 ++++++-- .../plugin_datastore/test_track_store.py | 14 +- .../track_geometry_store/test_pygeos_store.py | 123 +++++++++++------- 6 files changed, 214 insertions(+), 152 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 1455901ec..3e0933403 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -701,7 +701,9 @@ def track_ids(self) -> set[str]: @staticmethod @abstractmethod - def from_track_dataset(dataset: TrackDataset) -> "TrackGeometryDataset": + def from_track_dataset( + dataset: TrackDataset, offset: RelativeOffsetCoordinate + ) -> "TrackGeometryDataset": raise NotImplementedError @abstractmethod @@ -732,14 +734,11 @@ def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": raise NotImplementedError @abstractmethod - def intersecting_tracks( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. Args: sections (list[Section]): the list of sections to intersect. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: set[TrackId]: the track ids intersecting the given sections. @@ -748,7 +747,7 @@ def intersecting_tracks( @abstractmethod def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: """ Return the intersection points resulting from the tracks and the @@ -756,7 +755,6 @@ def intersection_points( Args: sections (list[Section]): the sections to intersect with. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId]]]: the intersection points. @@ -765,13 +763,12 @@ def intersection_points( @abstractmethod def contained_by_sections( - self, sections: list[Section], offset: RelativeOffsetCoordinate + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: """Return whether track coordinates are contained by the given sections. Args: sections (list[Section]): the sections. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask @@ -780,4 +777,6 @@ def contained_by_sections( raise NotImplementedError -TRACK_GEOMETRY_FACTORY = Callable[[TrackDataset], TrackGeometryDataset] +TRACK_GEOMETRY_FACTORY = Callable[ + [TrackDataset, RelativeOffsetCoordinate], TrackGeometryDataset +] diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 840027ce8..ac83c0d81 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -16,6 +16,7 @@ Track, TrackClassificationCalculator, TrackDataset, + TrackGeometryDataset, TrackHasNoDetectionError, TrackId, ) @@ -223,6 +224,8 @@ class PythonTrackDataset(TrackDataset): def __init__( self, values: Optional[dict[TrackId, Track]] = None, + geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] + | None = None, calculator: TrackClassificationCalculator = ByMaxConfidence(), track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( PygeosTrackGeometryDataset.from_track_dataset @@ -233,14 +236,21 @@ def __init__( self._tracks = values self._calculator = calculator self._track_geometry_factory = track_geometry_factory - self._track_geometry_dataset = track_geometry_factory(self) + if geometry_dataset is None: + self._geometry_dataset = dict[ + RelativeOffsetCoordinate, TrackGeometryDataset + ]() + else: + self._geometry_dataset = geometry_dataset @staticmethod def from_list( tracks: list[Track], calculator: TrackClassificationCalculator = ByMaxConfidence(), ) -> TrackDataset: - return PythonTrackDataset({track.id: track for track in tracks}, calculator) + return PythonTrackDataset( + {track.id: track for track in tracks}, calculator=calculator + ) def add_all(self, other: Iterable[Track]) -> "TrackDataset": if isinstance(other, PythonTrackDataset): @@ -306,7 +316,7 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: batch_size = ceil(dataset_size / batches) return [ - PythonTrackDataset(dict(batch), self._calculator) + PythonTrackDataset(dict(batch), calculator=self._calculator) for batch in batched(self._tracks.items(), batch_size) ] @@ -319,19 +329,43 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": for _id, track in self._tracks.items() if len(track.detections) >= length } - return PythonTrackDataset(filtered_tracks, self._calculator) + return PythonTrackDataset(filtered_tracks, calculator=self._calculator) def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> set[TrackId]: - return self._track_geometry_dataset.intersecting_tracks(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersecting_tracks(sections) + + def _get_geometry_dataset_for( + self, offset: RelativeOffsetCoordinate + ) -> TrackGeometryDataset: + """Retrieves track geometries for given offset. + + If offset does not exist, a new TrackGeometryDataset with the applied offset + will be created and saved. + + Args: + offset (RelativeOffsetCoordinate): the offset to retrieve track geometries + for. + + Returns: + TrackGeometryDataset: the track geometry dataset with the given offset + applied. + """ + if (geometry_dataset := self._geometry_dataset.get(offset, None)) is None: + geometry_dataset = self._track_geometry_factory(self, offset) + self._geometry_dataset[offset] = geometry_dataset + return geometry_dataset def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._track_geometry_dataset.intersection_points(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersection_points(sections) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - return self._track_geometry_dataset.contained_by_sections(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.contained_by_sections(sections) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index c9b0ef713..9d22dd984 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -99,28 +99,21 @@ class InvalidTrackGeometryDataset(Exception): class PygeosTrackGeometryDataset(TrackGeometryDataset): def __init__( self, - dataset: dict[RelativeOffsetCoordinate, DataFrame] | None = None, + offset: RelativeOffsetCoordinate, + dataset: DataFrame | None = None, ): - if dataset is not None: - self._check_is_valid(dataset) - self._dataset: dict[RelativeOffsetCoordinate, DataFrame] = dataset - else: + self._offset = offset + if dataset is None: self._dataset = self._create_empty() + else: + self._dataset = dataset - def _create_empty(self) -> dict[RelativeOffsetCoordinate, DataFrame]: - return {BASE_GEOMETRY: DataFrame(columns=COLUMNS)} - - def _check_is_valid( - self, dataset: dict[RelativeOffsetCoordinate, DataFrame] - ) -> None: - try: - dataset[BASE_GEOMETRY] - except KeyError: - raise InvalidTrackGeometryDataset(f"Missing entry for key {BASE_GEOMETRY}") + def _create_empty(self) -> DataFrame: + return DataFrame(columns=COLUMNS) @property def track_ids(self) -> set[str]: - return set(self._dataset[BASE_GEOMETRY].index) + return set(self._dataset.index) @property def empty(self) -> bool: @@ -129,21 +122,20 @@ def empty(self) -> bool: Returns: bool: True if dataset is empty, False otherwise. """ - return self._get_base_geometry().empty - - def _get_base_geometry(self) -> DataFrame: - return self._dataset[BASE_GEOMETRY] + return self._dataset.empty @staticmethod - def from_track_dataset(dataset: TrackDataset) -> TrackGeometryDataset: + def from_track_dataset( + dataset: TrackDataset, offset: RelativeOffsetCoordinate + ) -> TrackGeometryDataset: if len(dataset) == 0: - return PygeosTrackGeometryDataset() + return PygeosTrackGeometryDataset(offset) track_geom_df = DataFrame.from_dict( - PygeosTrackGeometryDataset._create_entries(dataset), + PygeosTrackGeometryDataset._create_entries(dataset, offset), columns=COLUMNS, orient=ORIENTATION_INDEX, ) - return PygeosTrackGeometryDataset({BASE_GEOMETRY: track_geom_df}) + return PygeosTrackGeometryDataset(offset, track_geom_df) @staticmethod def _create_entries( @@ -181,60 +173,48 @@ def _create_entries( def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: if self.empty: - new_entries = self._create_entries(tracks) + new_entries = self._create_entries(tracks, self._offset) return PygeosTrackGeometryDataset( - { - BASE_GEOMETRY: DataFrame.from_dict( - new_entries, orient=ORIENTATION_INDEX - ) - } + self._offset, DataFrame.from_dict(new_entries, orient=ORIENTATION_INDEX) ) - new_dataset = {} existing_entries = self.as_dict() - for offset in existing_entries.keys(): - new_entries = self._create_entries(tracks, offset) - for track_id, entry in new_entries.items(): - existing_entries[offset][track_id] = entry - new_dataset[offset] = DataFrame.from_dict( - existing_entries[offset], orient=ORIENTATION_INDEX - ) + new_entries = self._create_entries(tracks, self._offset) + for track_id, entry in new_entries.items(): + existing_entries[track_id] = entry + new_dataset = DataFrame.from_dict(existing_entries, orient=ORIENTATION_INDEX) - return PygeosTrackGeometryDataset(new_dataset) + return PygeosTrackGeometryDataset(self._offset, new_dataset) def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: - updated = {} - for offset, geometry_df in self._dataset.items(): - updated[offset] = geometry_df.drop( - index=[track_id.id for track_id in ids], errors="ignore" - ) - return PygeosTrackGeometryDataset(updated) + updated = self._dataset.drop( + index=[track_id.id for track_id in ids], errors="ignore" + ) + return PygeosTrackGeometryDataset(self._offset, updated) - def intersecting_tracks( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: intersecting_tracks = set() section_geoms = line_sections_to_pygeos_multi(sections) - track_df = self._get_track_geometries_for(offset) - track_df[INTERSECTS] = ( - track_df[GEOMETRY] + self._dataset[INTERSECTS] = ( + self._dataset[GEOMETRY] .apply(lambda line: intersects(line, section_geoms)) .map(any) .astype(bool) ) - track_ids = [TrackId(_id) for _id in track_df[track_df[INTERSECTS]].index] + track_ids = [ + TrackId(_id) for _id in self._dataset[self._dataset[INTERSECTS]].index + ] intersecting_tracks.update(track_ids) return intersecting_tracks def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: intersection_points = defaultdict(list) section_geoms = line_sections_to_pygeos_multi(sections) - track_df = self._get_track_geometries_for(offset) - track_df[INTERSECTIONS] = track_df[GEOMETRY].apply( + self._dataset[INTERSECTIONS] = self._dataset[GEOMETRY].apply( lambda line: [ (sections[index].id, ip) for index, ip in enumerate(intersection(line, section_geoms)) @@ -242,7 +222,7 @@ def intersection_points( ] ) intersections = ( - track_df[track_df[INTERSECTIONS].apply(lambda i: len(i) > 0)] + self._dataset[self._dataset[INTERSECTIONS].apply(lambda i: len(i) > 0)] .apply( lambda r: [ self._next_event( @@ -282,6 +262,12 @@ def _apply_offset( new_track_df[GEOMETRY] = new_track_df[GEOMETRY].apply( lambda geom: apply(geom, lambda coord: coord + coord * [offset.x, offset.y]) ) + new_track_df[PROJECTION] = new_track_df[GEOMETRY].apply( + lambda track_geom: [ + line_locate_point(track_geom, points(p)) + for p in get_coordinates(track_geom) + ] + ) return new_track_df def _next_event( @@ -300,7 +286,7 @@ def _next_event( ) def contained_by_sections( - self, sections: list[Section], offset: RelativeOffsetCoordinate + self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: contains_result: dict[ TrackId, list[tuple[SectionId, list[bool]]] @@ -308,8 +294,7 @@ def contained_by_sections( for _section in sections: section_geom = area_section_to_pygeos(_section) - track_df = self._get_track_geometries_for(offset) - contains_masks = track_df[GEOMETRY].apply( + contains_masks = self._dataset[GEOMETRY].apply( lambda line: [ contains(section_geom, points(p))[0] for p in get_coordinates(line) ] @@ -325,8 +310,4 @@ def contained_by_sections( return contains_result def as_dict(self) -> dict: - result = {} - for offset, track_geom_df in self._dataset.items(): - result[offset] = track_geom_df[COLUMNS].to_dict(orient=ORIENTATION_INDEX) - - return result + return self._dataset[COLUMNS].to_dict(orient=ORIENTATION_INDEX) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index fab77440c..06e7daff2 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -150,7 +150,8 @@ class PandasTrackDataset(TrackDataset): def __init__( self, dataset: DataFrame = DataFrame(), - geometry_dataset: TrackGeometryDataset | None = None, + geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] + | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( PygeosTrackGeometryDataset.from_track_dataset @@ -159,9 +160,10 @@ def __init__( self._dataset = dataset self._calculator = calculator self._track_geometry_factory = track_geometry_factory - # TODO: Re-use existing track geometries instead of creating new ones if geometry_dataset is None: - self._geometry_dataset = self._track_geometry_factory(self) + self._geometry_dataset = dict[ + RelativeOffsetCoordinate, TrackGeometryDataset + ]() else: self._geometry_dataset = geometry_dataset @@ -177,7 +179,8 @@ def from_list( @staticmethod def from_dataframe( tracks: DataFrame, - track_geometry_dataset: TrackGeometryDataset | None = None, + geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] + | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ) -> TrackDataset: if tracks.empty: @@ -195,7 +198,7 @@ def from_dataframe( valid_tracks = classified_tracks.loc[ classified_tracks[track.TRACK_ID].isin(valid_track_ids), : ] - return PandasTrackDataset(valid_tracks, geometry_dataset=track_geometry_dataset) + return PandasTrackDataset(valid_tracks, geometry_dataset=geometry_dataset) def add_all(self, other: Iterable[Track]) -> TrackDataset: new_tracks = self.__get_tracks(other) @@ -208,11 +211,19 @@ def add_all(self, other: Iterable[Track]) -> TrackDataset: new_dataset = pandas.concat([self._dataset, new_tracks]) new_track_ids = new_tracks[track.TRACK_ID].unique() new_tracks_df = new_dataset[new_dataset[track.TRACK_ID].isin(new_track_ids)] - updated_geometry_dataset = self._geometry_dataset.add_all( + updated_geometry_dataset = self._add_to_geometry_dataset( PandasTrackDataset.from_dataframe(new_tracks_df) ) return PandasTrackDataset.from_dataframe(new_dataset, updated_geometry_dataset) + def _add_to_geometry_dataset( + self, new_tracks: TrackDataset + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() + for offset, geometries in self._geometry_dataset.items(): + updated[offset] = geometries.add_all(new_tracks) + return updated + def get_all_ids(self) -> Iterable[TrackId]: return self._dataset[track.TRACK_ID].apply(lambda track_id: TrackId(track_id)) @@ -282,17 +293,41 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> set[TrackId]: - return self._geometry_dataset.intersecting_tracks(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersecting_tracks(sections) + + def _get_geometry_dataset_for( + self, offset: RelativeOffsetCoordinate + ) -> TrackGeometryDataset: + """Retrieves track geometries for given offset. + + If offset does not exist, a new TrackGeometryDataset with the applied offset + will be created and saved. + + Args: + offset (RelativeOffsetCoordinate): the offset to retrieve track geometries + for. + + Returns: + TrackGeometryDataset: the track geometry dataset with the given offset + applied. + """ + if (geometry_dataset := self._geometry_dataset.get(offset, None)) is None: + geometry_dataset = self._track_geometry_factory(self, offset) + self._geometry_dataset[offset] = geometry_dataset + return geometry_dataset def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._geometry_dataset.intersection_points(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersection_points(sections) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - return self._geometry_dataset.contained_by_sections(sections, offset) + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.contained_by_sections(sections) def _assign_track_classification( diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 4f1e5549b..33eba02f0 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -122,12 +122,7 @@ def test_add_all(self) -> None: ) for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) - assert merged._geometry_dataset.track_ids == { - first_track.id.id, - second_track.id.id, - third_track.id.id, - } - assert 1 == len(dataset.as_list()) + assert merged._geometry_dataset == {} def test_add_two_existing_pandas_datasets(self) -> None: first_track = self.__build_track("1") @@ -139,12 +134,7 @@ def test_add_two_existing_pandas_datasets(self) -> None: for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) - - # TODO: Is it enough to check for track ids in the geometry dataset? - assert merged._geometry_dataset.track_ids == { - first_track.id.id, - second_track.id.id, - } + assert merged._geometry_dataset == {} def __build_track(self, track_id: str, length: int = 5) -> Track: builder = TrackBuilder() diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index b9adf0d57..b4ff865cc 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -4,7 +4,7 @@ import pytest from pandas import DataFrame -from pygeos import Geometry, get_coordinates, line_locate_point, linestrings, points +from pygeos import get_coordinates, line_locate_point, points from pytest_benchmark.fixture import BenchmarkFixture from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate @@ -24,6 +24,7 @@ PROJECTION, TRACK_ID, PygeosTrackGeometryDataset, + create_pygeos_track, ) from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser, OttrkParser @@ -31,10 +32,6 @@ from tests.conftest import TrackBuilder -def create_pygeos_track(track: Track) -> Geometry: - return linestrings([(detection.x, detection.y) for detection in track.detections]) - - def create_track_dataset(tracks: list[Track]) -> TrackDataset: dataset = MagicMock() dataset.__iter__.return_value = iter(tracks) @@ -42,22 +39,23 @@ def create_track_dataset(tracks: list[Track]) -> TrackDataset: return dataset -def create_geometry_dataset_from(tracks: Iterable[Track]) -> PygeosTrackGeometryDataset: +def create_geometry_dataset_from( + tracks: Iterable[Track], offset: RelativeOffsetCoordinate +) -> PygeosTrackGeometryDataset: entries = [] for track in tracks: _id = track.id.id - geometry = create_pygeos_track(track) + geometry = create_pygeos_track(track, offset) projection = [ line_locate_point(geometry, points(p)) for p in get_coordinates(geometry) ] entries.append((_id, geometry, projection)) return PygeosTrackGeometryDataset( - { - BASE_GEOMETRY: DataFrame.from_records( - entries, - columns=[TRACK_ID, GEOMETRY, PROJECTION], - ).set_index(TRACK_ID) - } + offset, + DataFrame.from_records( + entries, + columns=[TRACK_ID, GEOMETRY, PROJECTION], + ).set_index(TRACK_ID), ) @@ -300,10 +298,7 @@ def assert_track_geometry_dataset_equals( ) -> None: assert isinstance(to_compare, PygeosTrackGeometryDataset) assert isinstance(other, PygeosTrackGeometryDataset) - assert to_compare._dataset.keys() == other._dataset.keys() # noqa - - for offset, track_geom in to_compare._dataset.items(): # noqa - assert track_geom.equals(other._dataset[offset]) # noqa + assert to_compare._dataset.equals(other._dataset) # noqa class TestPygeosTrackGeometryDataset: @@ -322,58 +317,78 @@ def simple_track(self) -> Track: def test_from_track_dataset(self, simple_track: Track) -> None: track_dataset = create_track_dataset([simple_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + dataset=track_dataset, offset=BASE_GEOMETRY + ) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) - expected = create_geometry_dataset_from([simple_track]) + expected = create_geometry_dataset_from([simple_track], BASE_GEOMETRY) assert_track_geometry_dataset_equals(geometry_dataset, expected) def test_add_all_on_empty_dataset( self, first_track: Track, second_track: Track ) -> None: track_dataset = create_track_dataset([]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.add_all([first_track, second_track]) - expected = create_geometry_dataset_from([first_track, second_track]) + expected = create_geometry_dataset_from( + [first_track, second_track], BASE_GEOMETRY + ) assert_track_geometry_dataset_equals(result, expected) def test_add_all_on_filled_dataset( self, first_track: Track, second_track: Track ) -> None: track_dataset = create_track_dataset([first_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.add_all([second_track]) - expected = create_geometry_dataset_from([first_track, second_track]) + expected = create_geometry_dataset_from( + [first_track, second_track], BASE_GEOMETRY + ) assert_track_geometry_dataset_equals(result, expected) def test_add_all_merge_track( self, first_track: Track, first_track_merged: Track, second_track: Track ) -> None: track_dataset = create_track_dataset([first_track, second_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.add_all([first_track_merged]) - expected = create_geometry_dataset_from([first_track_merged, second_track]) + expected = create_geometry_dataset_from( + [first_track_merged, second_track], BASE_GEOMETRY + ) assert_track_geometry_dataset_equals(result, expected) def test_remove_from_filled_dataset( self, first_track: Track, second_track: Track ) -> None: track_dataset = create_track_dataset([first_track, second_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.remove([first_track.id]) - expected = create_geometry_dataset_from([second_track]) + expected = create_geometry_dataset_from([second_track], BASE_GEOMETRY) assert_track_geometry_dataset_equals(result, expected) def test_remove_from_empty_dataset(self, first_track: Track) -> None: - geometry_dataset = PygeosTrackGeometryDataset() + geometry_dataset = PygeosTrackGeometryDataset(BASE_GEOMETRY) result = geometry_dataset.remove([first_track.id]) - assert_track_geometry_dataset_equals(result, PygeosTrackGeometryDataset()) + assert_track_geometry_dataset_equals( + result, PygeosTrackGeometryDataset(BASE_GEOMETRY) + ) def test_remove_missing(self, first_track: Track, second_track: Track) -> None: track_dataset = create_track_dataset([first_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.remove([second_track.id]) - expected = create_geometry_dataset_from([first_track]) + expected = create_geometry_dataset_from([first_track], BASE_GEOMETRY) assert_track_geometry_dataset_equals(result, expected) def test_intersection_points( @@ -396,8 +411,10 @@ def test_intersection_points( track_dataset = create_track_dataset( [single_detection_track, not_intersecting_track, first_track, second_track] ) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) - result = geometry_dataset.intersection_points(sections, BASE_GEOMETRY) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) + result = geometry_dataset.intersection_points(sections) assert result == { first_track.id: [ (first_section.id, IntersectionPoint(1)), @@ -431,41 +448,44 @@ def test_intersecting_tracks( track_dataset = create_track_dataset( [single_detection_track, not_intersecting_track, first_track, second_track] ) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) - result = geometry_dataset.intersecting_tracks(sections, BASE_GEOMETRY) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) + result = geometry_dataset.intersecting_tracks(sections) assert result == {first_track.id, second_track.id} def test_as_dict(self, first_track: Track, second_track: Track) -> None: tracks = [first_track, second_track] track_dataset = create_track_dataset(tracks) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) result = geometry_dataset.as_dict() - expected = { - BASE_GEOMETRY: create_geometry_dataset_from(tracks) - ._dataset[BASE_GEOMETRY][COLUMNS] + expected = ( + create_geometry_dataset_from(tracks, BASE_GEOMETRY) + ._dataset[COLUMNS] .to_dict(orient="index") - } + ) assert result == expected - def test_get_base_geometry(self) -> None: - base_geometry = Mock() - geometry_dataset = PygeosTrackGeometryDataset({BASE_GEOMETRY: base_geometry}) - assert geometry_dataset._get_base_geometry() == base_geometry - def test_empty_on_empty_dataset(self) -> None: - geometry_dataset = PygeosTrackGeometryDataset() + geometry_dataset = PygeosTrackGeometryDataset(BASE_GEOMETRY) assert geometry_dataset.empty def test_empty_on_filled_dataset(self, first_track: Track) -> None: track_dataset = create_track_dataset([first_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) assert not geometry_dataset.empty def test_get_track_ids(self, first_track: Track) -> None: track_dataset = create_track_dataset([first_track]) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) assert geometry_dataset.track_ids == {first_track.id.id} @@ -480,9 +500,11 @@ def test_contained_by_sections( track_dataset = create_track_dataset( [not_intersecting_track, first_track, second_track] ) - geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset(track_dataset) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) result = geometry_dataset.contained_by_sections( - [not_intersecting_area_section, area_section], BASE_GEOMETRY + [not_intersecting_area_section, area_section] ) expected = { first_track.id: [ @@ -514,6 +536,7 @@ def sections(self, test_data_dir: Path) -> Iterable[Section]: sections, flows = flow_parser.parse(flow_file) return sections + @pytest.mark.skip def test_profile( self, benchmark: BenchmarkFixture, From 52cf2c62aa45c5751ac1ab9961839dbe1402ff36 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:42:37 +0100 Subject: [PATCH 045/107] Set classification only once per track --- .../plugin_intersect/shapely/create_intersection_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index 241a74320..68b7cff25 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -137,9 +137,9 @@ def __do_intersect( "Track not found. Unable to create intersection event " f"for track {track_id}." ) + event_builder.add_road_user_type(track.classification) for section_id, intersection_point in intersection_points: event_builder.add_section_id(section_id) - event_builder.add_road_user_type(track.classification) detection = track.detections[intersection_point.index] current_coord = detection.get_coordinate(offset) prev_coord = track.detections[ From 1f6eb37ea0f23f088dbb7782d53d677c3cf42ad0 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:43:36 +0100 Subject: [PATCH 046/107] Use coordinate with offset applied as event coordinate --- .../plugin_intersect/shapely/create_intersection_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index 68b7cff25..d53aeb264 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -153,7 +153,7 @@ def __do_intersect( ) event_builder.add_event_type(EventType.SECTION_ENTER) event_builder.add_direction_vector(direction_vector) - event_builder.add_event_coordinate(detection.x, detection.y) + event_builder.add_event_coordinate(current_coord.x, current_coord.y) events.append(event_builder.create_event(detection)) return events From 525ad2b3da189a9e0bf5ab67d401b5a28ee8cf37 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:44:02 +0100 Subject: [PATCH 047/107] Fix unit test --- .../plugin_intersect/shapely/test_create_intersection_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py index 3a69cafad..5f5951e39 100644 --- a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py +++ b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py @@ -282,7 +282,7 @@ def test_case_closed_track_line_section( (section.id, IntersectionPoint(4)), ] } - track_dataset.get_for.return_value = track + track_dataset.get_for.return_value = closed_track expected_event_coords = [ _ExpectedEventCoord(2, 2.0, 2.0), _ExpectedEventCoord(4, 1.0, 1.0), From ef10f28994fff0d321acb1bf8384e47df2887265 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:04:45 +0100 Subject: [PATCH 048/107] Fix unit test --- .../plugin_parser/test_otvision_parser.py | 13 +++++++------ tests/conftest.py | 7 ++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py index 7aee5e7e7..d8761fc8c 100644 --- a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py @@ -69,7 +69,7 @@ _write_bz2, _write_json, ) -from tests.conftest import TrackBuilder +from tests.conftest import TrackBuilder, assert_track_datasets_equal @pytest.fixture @@ -264,7 +264,9 @@ def test_parse_ottrk_sample( expected_detection_classes = frozenset( ["person", "bus", "boat", "truck", "car", "motorcycle", "bicycle", "train"] ) - assert parse_result.tracks == PythonTrackDataset.from_list([expected_track]) + assert_track_datasets_equal( + parse_result.tracks, PythonTrackDataset.from_list([expected_track]) + ) assert parse_result.metadata.detection_classes == expected_detection_classes ottrk_file.unlink() @@ -331,9 +333,8 @@ def test_parse_tracks( expected_sorted = PythonTrackDataset.from_list( [track_builder_setup_with_sample_data.build_track()] ) - - assert expected_sorted == result_sorted_input - assert expected_sorted == result_unsorted_input + assert_track_datasets_equal(result_sorted_input, expected_sorted) + assert_track_datasets_equal(result_unsorted_input, expected_sorted) def test_parse_tracks_merge_with_existing( self, @@ -370,7 +371,7 @@ def test_parse_tracks_merge_with_existing( expected_sorted = PythonTrackDataset.from_list([merged_track]) - assert expected_sorted == result_sorted_input + assert_track_datasets_equal(result_sorted_input, expected_sorted) @pytest.mark.parametrize( "track_length_limit", diff --git a/tests/conftest.py b/tests/conftest.py index 36b06a279..e479ff177 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from OTAnalytics.domain.event import Event, EventType from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence from OTAnalytics.plugin_parser import ottrk_dataformat @@ -415,6 +415,11 @@ def assert_equal_track_properties(actual: Track, expected: Track) -> None: assert_equal_detection_properties(second_detection, first_detection) +def assert_track_datasets_equal(actual: TrackDataset, expected: TrackDataset) -> None: + for actual_track, expected_track in zip(actual.as_list(), expected.as_list()): + assert_equal_track_properties(actual_track, expected_track) + + def append_sample_data( track_builder: TrackBuilder, frame_offset: int = 0, From 44cc70863edb5e4d404d35e01d99378c2dbf287f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:16:06 +0100 Subject: [PATCH 049/107] Remove unused methods --- .../track_geometry_store/pygeos_store.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 9d22dd984..67dfa640b 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -6,7 +6,6 @@ from pandas import DataFrame from pygeos import ( Geometry, - apply, contains, geometrycollections, get_coordinates, @@ -244,32 +243,6 @@ def intersection_points( return intersection_points - def _get_track_geometries_for(self, offset: RelativeOffsetCoordinate) -> DataFrame: - if (track_df := self._dataset.get(offset, None)) is None: - self._create_tracks_for(offset) - track_df = self._dataset[offset] - return track_df - - def _create_tracks_for(self, offset: RelativeOffsetCoordinate) -> None: - base_track_geometry = self._dataset[BASE_GEOMETRY] - self._dataset[offset] = self._apply_offset(base_track_geometry, offset) - - @staticmethod - def _apply_offset( - base_track_geometries: DataFrame, offset: RelativeOffsetCoordinate - ) -> DataFrame: - new_track_df = base_track_geometries.copy() - new_track_df[GEOMETRY] = new_track_df[GEOMETRY].apply( - lambda geom: apply(geom, lambda coord: coord + coord * [offset.x, offset.y]) - ) - new_track_df[PROJECTION] = new_track_df[GEOMETRY].apply( - lambda track_geom: [ - line_locate_point(track_geom, points(p)) - for p in get_coordinates(track_geom) - ] - ) - return new_track_df - def _next_event( self, track_id: str, From a45eae0958a11c4d2dd38d2f4f6712bbabe452bc Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:42:42 +0100 Subject: [PATCH 050/107] Remove IntersectionVisitor --- OTAnalytics/domain/section.py | 22 +------------------ .../shapely/create_intersection_events.py | 10 ++------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/OTAnalytics/domain/section.py b/OTAnalytics/domain/section.py index 0123708be..7bcaf426b 100644 --- a/OTAnalytics/domain/section.py +++ b/OTAnalytics/domain/section.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Any, Callable, Generic, Iterable, Optional, TypeVar +from typing import Any, Callable, Iterable, Optional, TypeVar from OTAnalytics.application.config import CUTTING_SECTION_MARKER from OTAnalytics.domain.common import DataclassValidation @@ -205,10 +205,6 @@ def _serialize_relative_offset_coordinates(self) -> dict[str, dict]: for event_type, offset in self.relative_offset_coordinates.items() } - @abstractmethod - def accept(self, visitor: "IntersectionVisitor[T]") -> list[T]: - raise NotImplementedError - @dataclass(frozen=True) class LineSection(Section): @@ -291,9 +287,6 @@ def get_type(self) -> SectionType: def _is_cutting_section(self) -> bool: return self.name.startswith(CUTTING_SECTION_MARKER) - def accept(self, visitor: "IntersectionVisitor[T]") -> list[T]: - return visitor.intersect_line_section(self) - @dataclass(frozen=True) class Area(Section): @@ -359,9 +352,6 @@ def to_dict(self) -> dict: def get_type(self) -> SectionType: return SectionType.AREA - def accept(self, visitor: "IntersectionVisitor[T]") -> list[T]: - return visitor.intersect_area_section(self) - class MissingSection(Exception): pass @@ -491,13 +481,3 @@ def clear(self) -> None: self._repository_content_observers.notify( SectionRepositoryEvent.create_removed(removed) ) - - -class IntersectionVisitor(ABC, Generic[T]): - @abstractmethod - def intersect_area_section(self, section: Area) -> list[T]: - raise NotImplementedError - - @abstractmethod - def intersect_line_section(self, section: LineSection) -> list[T]: - raise NotImplementedError diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index d53aeb264..4dca0217f 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -17,7 +17,7 @@ calculate_direction_vector, ) from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy -from OTAnalytics.domain.section import Area, IntersectionVisitor, LineSection, Section +from OTAnalytics.domain.section import Area, LineSection, Section from OTAnalytics.domain.track import Track, TrackDataset, TrackId from OTAnalytics.domain.types import EventType @@ -265,7 +265,7 @@ def __do_intersect( return events -class ShapelyCreateIntersectionEvents(IntersectionVisitor[Event]): +class ShapelyCreateIntersectionEvents: def __init__( self, intersect_line_section: Intersector, @@ -295,12 +295,6 @@ def create(self) -> list[Event]: ) return events - def intersect_line_section(self, section: LineSection) -> list[Event]: - raise NotImplementedError - - def intersect_area_section(self, section: Area) -> list[Event]: - raise NotImplementedError - class ShapelyRunIntersect(RunIntersect): def __init__( From c06544ecebc6564850563e76a266c2c48bae3e55 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 18:49:47 +0100 Subject: [PATCH 051/107] Extend unit test for contained_by_sections --- .../track_geometry_store/test_pygeos_store.py | 123 +++++++++++++++--- .../test_create_intersection_events.py | 86 ------------ tests/conftest.py | 94 ++++++++++++- 3 files changed, 198 insertions(+), 105 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index b4ff865cc..b5ae20b67 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from pathlib import Path from typing import Iterable from unittest.mock import MagicMock, Mock @@ -39,6 +40,17 @@ def create_track_dataset(tracks: list[Track]) -> TrackDataset: return dataset +def create_line_section( + section_id: str, coordinates: list[tuple[float, float]] +) -> Section: + section = Mock(spec=LineSection) + section.get_coordinates.return_value = [ + Coordinate(coord[0], coord[1]) for coord in coordinates + ] + section.id = SectionId(section_id) + return section + + def create_geometry_dataset_from( tracks: Iterable[Track], offset: RelativeOffsetCoordinate ) -> PygeosTrackGeometryDataset: @@ -301,6 +313,95 @@ def assert_track_geometry_dataset_equals( assert to_compare._dataset.equals(other._dataset) # noqa +@dataclass +class ContainedBySectionTestCase: + tracks: list[Track] + sections: list[Section] + expected_result: dict[TrackId, list[tuple[SectionId, list[bool]]]] + + +@pytest.fixture +def contained_by_section_test_case( + straight_track: Track, complex_track: Track, closed_track: Track +) -> ContainedBySectionTestCase: + # Straight track starts outside section + first_section = create_line_section( + "1", [(1.5, 0.5), (1.5, 1.5), (2.5, 1.5), (2.5, 0.5), (1.5, 0.5)] + ) + # Straight track starts inside section + second_section = create_line_section( + "2", [(0.5, 0.5), (0.5, 1.5), (1.5, 1.5), (1.5, 0.5), (0.5, 0.5)] + ) + # Straight track is inside section + third_section = create_line_section( + "3", [(0.0, 0.0), (0.0, 2.0), (4.0, 2.0), (4.0, 0.0), (0.0, 0.0)] + ) + # Straight track starts outside stays inside section + fourth_section = create_line_section( + "4", [(1.5, 0.5), (1.5, 1.5), (4.0, 1.5), (4.0, 0.5), (1.5, 0.5)] + ) + # Complex track starts outside section with multiple intersections + fifth_section = create_line_section( + "5", + [(1.5, 0.5), (1.5, 2.5), (2.5, 2.5), (2.5, 0.5), (1.5, 0.5)], + ) + # Complex track starts inside section with multiple intersections + sixth_section = create_line_section( + "6", [(0.5, 0.5), (0.5, 2.5), (1.5, 2.5), (1.5, 0.5), (0.5, 0.5)] + ) + # Closed track + seventh_section = create_line_section( + "7", [(1.5, 0.5), (1.5, 2.0), (2.5, 2.0), (2.5, 0.5), (1.5, 0.5)] + ) + # Not contained track + eighth_section = create_line_section( + "not-contained", [(3.0, 1.0), (3.0, 2.0), (4.0, 2.0), (4.0, 1.0), (3.0, 1.0)] + ) + expected = { + straight_track.id: [ + (first_section.id, [False, True, False]), + (second_section.id, [True, False, False]), + (third_section.id, [True, True, True]), + (fourth_section.id, [False, True, True]), + (fifth_section.id, [False, True, False]), + (sixth_section.id, [True, False, False]), + (seventh_section.id, [False, True, False]), + ], + complex_track.id: [ + (first_section.id, [False, True, False, False, False, False]), + (second_section.id, [True, False, False, False, False, False]), + (third_section.id, [True, True, True, True, False, False]), + (fourth_section.id, [False, True, False, False, False, False]), + (fifth_section.id, [False, True, True, False, False, True]), + (sixth_section.id, [True, False, False, True, True, False]), + (seventh_section.id, [False, True, True, False, False, False]), + ], + closed_track.id: [ + (first_section.id, [False, True, False, False, False]), + (second_section.id, [True, False, False, False, True]), + (third_section.id, [True, True, False, False, True]), + (fourth_section.id, [False, True, False, False, False]), + (fifth_section.id, [False, True, True, False, False]), + (sixth_section.id, [True, False, False, True, True]), + (seventh_section.id, [False, True, False, False, False]), + ], + } + return ContainedBySectionTestCase( + [straight_track, complex_track, closed_track], + [ + first_section, + second_section, + third_section, + fourth_section, + fifth_section, + sixth_section, + seventh_section, + eighth_section, + ], + expected, + ) + + class TestPygeosTrackGeometryDataset: @pytest.fixture def simple_track(self) -> Track: @@ -491,30 +592,16 @@ def test_get_track_ids(self, first_track: Track) -> None: def test_contained_by_sections( self, - first_track: Track, - second_track: Track, - not_intersecting_track: Track, - area_section: Section, - not_intersecting_area_section: Section, + contained_by_section_test_case: ContainedBySectionTestCase, ) -> None: - track_dataset = create_track_dataset( - [not_intersecting_track, first_track, second_track] - ) + track_dataset = create_track_dataset(contained_by_section_test_case.tracks) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( track_dataset, BASE_GEOMETRY ) result = geometry_dataset.contained_by_sections( - [not_intersecting_area_section, area_section] + contained_by_section_test_case.sections ) - expected = { - first_track.id: [ - (area_section.id, [False, True, False, False, False]), - ], - second_track.id: [ - (area_section.id, [False, True, False, False, False]), - ], - } - assert result == expected + assert result == contained_by_section_test_case.expected_result class TestProfiling: diff --git a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py index 5f5951e39..f6671ab82 100644 --- a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py +++ b/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py @@ -208,42 +208,6 @@ def track(track_builder: TrackBuilder) -> Track: return track_builder.build_track() -@pytest.fixture -def closed_track(track_builder: TrackBuilder) -> Track: - classification = "car" - track_id = "2" - - track_builder.add_track_class(classification) - track_builder.add_detection_class(classification) - track_builder.add_track_id(track_id) - - track_builder.add_frame(1) - track_builder.add_second(1) - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.append_detection() - - track_builder.add_frame(2) - track_builder.add_second(2) - track_builder.add_xy_bbox(2.0, 1.0) - track_builder.append_detection() - - track_builder.add_frame(3) - track_builder.add_second(3) - track_builder.add_xy_bbox(2.0, 2.0) - track_builder.append_detection() - - track_builder.add_frame(5) - track_builder.add_second(5) - track_builder.add_xy_bbox(1.0, 2.0) - track_builder.append_detection() - - track_builder.add_frame(5) - track_builder.add_second(5) - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.append_detection() - return track_builder.build_track() - - @pytest.fixture def test_case_track_line_section(track: Track) -> _TestCase: offset = (0, 0.5) @@ -343,56 +307,6 @@ class TestShapelyIntersectAreaByTrackPoints: def _create_intersector(self) -> ShapelyIntersectAreaByTrackPoints: return ShapelyIntersectAreaByTrackPoints() - @pytest.fixture - def straight_track(self, track_builder: TrackBuilder) -> Track: - track_builder.add_wh_bbox(0.5, 0.5) - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 1.0) - track_builder.add_frame(2) - track_builder.add_microsecond(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(3.0, 1.0) - track_builder.add_frame(3) - track_builder.add_microsecond(2) - track_builder.append_detection() - - return track_builder.build_track() - - @pytest.fixture - def complex_track(self, track_builder: TrackBuilder) -> Track: - track_builder.add_xy_bbox(1.0, 1.0) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 1.0) - track_builder.add_frame(2) - track_builder.add_microsecond(1) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 1.5) - track_builder.add_frame(3) - track_builder.add_microsecond(2) - track_builder.append_detection() - - track_builder.add_xy_bbox(1.0, 1.5) - track_builder.add_frame(4) - track_builder.add_microsecond(3) - track_builder.append_detection() - - track_builder.add_xy_bbox(1.0, 2.0) - track_builder.add_frame(5) - track_builder.add_microsecond(4) - track_builder.append_detection() - - track_builder.add_xy_bbox(2.0, 2.0) - track_builder.add_frame(5) - track_builder.add_microsecond(4) - track_builder.append_detection() - - return track_builder.build_track() - @pytest.fixture def test_case_track_starts_outside_section( self, straight_track: Track diff --git a/tests/conftest.py b/tests/conftest.py index e479ff177..4e019d1da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,10 +6,11 @@ import pytest -from OTAnalytics.domain.event import Event, EventType +from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence from OTAnalytics.plugin_parser import ottrk_dataformat @@ -391,6 +392,97 @@ def event_builder() -> EventBuilder: return EventBuilder() +@pytest.fixture +def straight_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("straight-track") + track_builder.add_wh_bbox(0.5, 0.5) + track_builder.add_xy_bbox(1.0, 1.0) + track_builder.append_detection() + + track_builder.add_xy_bbox(2.0, 1.0) + track_builder.add_frame(2) + track_builder.add_microsecond(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(3.0, 1.0) + track_builder.add_frame(3) + track_builder.add_microsecond(2) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def complex_track() -> Track: + track_builder = TrackBuilder() + track_builder.add_track_id("complex-track") + track_builder.add_xy_bbox(1.0, 1.0) + track_builder.append_detection() + + track_builder.add_xy_bbox(2.0, 1.0) + track_builder.add_frame(2) + track_builder.add_microsecond(1) + track_builder.append_detection() + + track_builder.add_xy_bbox(2.0, 1.5) + track_builder.add_frame(3) + track_builder.add_microsecond(2) + track_builder.append_detection() + + track_builder.add_xy_bbox(1.0, 1.5) + track_builder.add_frame(4) + track_builder.add_microsecond(3) + track_builder.append_detection() + + track_builder.add_xy_bbox(1.0, 2.0) + track_builder.add_frame(5) + track_builder.add_microsecond(4) + track_builder.append_detection() + + track_builder.add_xy_bbox(2.0, 2.0) + track_builder.add_frame(5) + track_builder.add_microsecond(4) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def closed_track() -> Track: + classification = "car" + track_builder = TrackBuilder() + track_builder.add_track_id("closed-track") + track_builder.add_track_class(classification) + track_builder.add_detection_class(classification) + + track_builder.add_frame(1) + track_builder.add_second(1) + track_builder.add_xy_bbox(1.0, 1.0) + track_builder.append_detection() + + track_builder.add_frame(2) + track_builder.add_second(2) + track_builder.add_xy_bbox(2.0, 1.0) + track_builder.append_detection() + + track_builder.add_frame(3) + track_builder.add_second(3) + track_builder.add_xy_bbox(2.0, 2.0) + track_builder.append_detection() + + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.add_xy_bbox(1.0, 2.0) + track_builder.append_detection() + + track_builder.add_frame(5) + track_builder.add_second(5) + track_builder.add_xy_bbox(1.0, 1.0) + track_builder.append_detection() + return track_builder.build_track() + + def assert_equal_detection_properties(actual: Detection, expected: Detection) -> None: assert expected.classification == actual.classification assert expected.confidence == actual.confidence From 215b7a0d48699087b44024c0ae6895c32b51815a Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 29 Nov 2023 19:12:13 +0100 Subject: [PATCH 052/107] Test offset applied when creating or adding to PygeosTrackGeometryDataset --- .../track_geometry_store/test_pygeos_store.py | 70 ++++++++++++------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index b5ae20b67..31a2743dd 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -8,6 +8,7 @@ from pygeos import get_coordinates, line_locate_point, points from pytest_benchmark.fixture import BenchmarkFixture +from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Area, LineSection, Section, SectionId from OTAnalytics.domain.track import ( @@ -408,88 +409,103 @@ def simple_track(self) -> Track: first_detection = Mock() first_detection.x = 1 first_detection.y = 0 + first_detection.w = 4 + first_detection.h = 4 second_detection = Mock() second_detection.x = 2 second_detection.y = 0 + second_detection.w = 5 + second_detection.h = 5 simple_track = Mock() simple_track.id = TrackId("1") simple_track.detections = [first_detection, second_detection] return simple_track - def test_from_track_dataset(self, simple_track: Track) -> None: + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) + def test_from_track_dataset( + self, simple_track: Track, offset: RelativeOffsetCoordinate + ) -> None: track_dataset = create_track_dataset([simple_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - dataset=track_dataset, offset=BASE_GEOMETRY + dataset=track_dataset, offset=offset ) assert isinstance(geometry_dataset, PygeosTrackGeometryDataset) - expected = create_geometry_dataset_from([simple_track], BASE_GEOMETRY) + expected = create_geometry_dataset_from([simple_track], offset) assert_track_geometry_dataset_equals(geometry_dataset, expected) + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) def test_add_all_on_empty_dataset( - self, first_track: Track, second_track: Track + self, first_track: Track, second_track: Track, offset: RelativeOffsetCoordinate ) -> None: track_dataset = create_track_dataset([]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - track_dataset, BASE_GEOMETRY + track_dataset, offset ) result = geometry_dataset.add_all([first_track, second_track]) - expected = create_geometry_dataset_from( - [first_track, second_track], BASE_GEOMETRY - ) + expected = create_geometry_dataset_from([first_track, second_track], offset) assert_track_geometry_dataset_equals(result, expected) + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) def test_add_all_on_filled_dataset( - self, first_track: Track, second_track: Track + self, first_track: Track, second_track: Track, offset: RelativeOffsetCoordinate ) -> None: track_dataset = create_track_dataset([first_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - track_dataset, BASE_GEOMETRY + track_dataset, offset ) result = geometry_dataset.add_all([second_track]) - expected = create_geometry_dataset_from( - [first_track, second_track], BASE_GEOMETRY - ) + expected = create_geometry_dataset_from([first_track, second_track], offset) assert_track_geometry_dataset_equals(result, expected) + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) def test_add_all_merge_track( - self, first_track: Track, first_track_merged: Track, second_track: Track + self, + first_track: Track, + first_track_merged: Track, + second_track: Track, + offset: RelativeOffsetCoordinate, ) -> None: track_dataset = create_track_dataset([first_track, second_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - track_dataset, BASE_GEOMETRY + track_dataset, offset ) result = geometry_dataset.add_all([first_track_merged]) expected = create_geometry_dataset_from( - [first_track_merged, second_track], BASE_GEOMETRY + [first_track_merged, second_track], offset ) assert_track_geometry_dataset_equals(result, expected) + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) def test_remove_from_filled_dataset( - self, first_track: Track, second_track: Track + self, first_track: Track, second_track: Track, offset: RelativeOffsetCoordinate ) -> None: track_dataset = create_track_dataset([first_track, second_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - track_dataset, BASE_GEOMETRY + track_dataset, offset ) result = geometry_dataset.remove([first_track.id]) - expected = create_geometry_dataset_from([second_track], BASE_GEOMETRY) + expected = create_geometry_dataset_from([second_track], offset) assert_track_geometry_dataset_equals(result, expected) - def test_remove_from_empty_dataset(self, first_track: Track) -> None: - geometry_dataset = PygeosTrackGeometryDataset(BASE_GEOMETRY) + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) + def test_remove_from_empty_dataset( + self, first_track: Track, offset: RelativeOffsetCoordinate + ) -> None: + geometry_dataset = PygeosTrackGeometryDataset(offset) result = geometry_dataset.remove([first_track.id]) - assert_track_geometry_dataset_equals( - result, PygeosTrackGeometryDataset(BASE_GEOMETRY) - ) + assert_track_geometry_dataset_equals(result, PygeosTrackGeometryDataset(offset)) - def test_remove_missing(self, first_track: Track, second_track: Track) -> None: + @pytest.mark.parametrize("offset", [BASE_GEOMETRY, DEFAULT_TRACK_OFFSET]) + def test_remove_missing( + self, first_track: Track, second_track: Track, offset: RelativeOffsetCoordinate + ) -> None: track_dataset = create_track_dataset([first_track]) geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( - track_dataset, BASE_GEOMETRY + track_dataset, offset ) result = geometry_dataset.remove([second_track.id]) - expected = create_geometry_dataset_from([first_track], BASE_GEOMETRY) + expected = create_geometry_dataset_from([first_track], offset) assert_track_geometry_dataset_equals(result, expected) def test_intersection_points( From 17c4ad43c9f2ec9de48d2228bc644a8d00323d4b Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:25:59 +0100 Subject: [PATCH 053/107] Rename variable --- .../plugin_datastore/python_track_store.py | 12 ++++++------ OTAnalytics/plugin_datastore/track_store.py | 16 ++++++++-------- .../plugin_datastore/test_track_store.py | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index ac83c0d81..0e7d2d1cb 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -224,7 +224,7 @@ class PythonTrackDataset(TrackDataset): def __init__( self, values: Optional[dict[TrackId, Track]] = None, - geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] + geometry_datasets: dict[RelativeOffsetCoordinate, TrackGeometryDataset] | None = None, calculator: TrackClassificationCalculator = ByMaxConfidence(), track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( @@ -236,12 +236,12 @@ def __init__( self._tracks = values self._calculator = calculator self._track_geometry_factory = track_geometry_factory - if geometry_dataset is None: - self._geometry_dataset = dict[ + if geometry_datasets is None: + self._geometry_datasets = dict[ RelativeOffsetCoordinate, TrackGeometryDataset ]() else: - self._geometry_dataset = geometry_dataset + self._geometry_datasets = geometry_datasets @staticmethod def from_list( @@ -353,9 +353,9 @@ def _get_geometry_dataset_for( TrackGeometryDataset: the track geometry dataset with the given offset applied. """ - if (geometry_dataset := self._geometry_dataset.get(offset, None)) is None: + if (geometry_dataset := self._geometry_datasets.get(offset, None)) is None: geometry_dataset = self._track_geometry_factory(self, offset) - self._geometry_dataset[offset] = geometry_dataset + self._geometry_datasets[offset] = geometry_dataset return geometry_dataset def intersection_points( diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 06e7daff2..2c95c111b 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -150,7 +150,7 @@ class PandasTrackDataset(TrackDataset): def __init__( self, dataset: DataFrame = DataFrame(), - geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] + geometry_datasets: dict[RelativeOffsetCoordinate, TrackGeometryDataset] | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( @@ -160,12 +160,12 @@ def __init__( self._dataset = dataset self._calculator = calculator self._track_geometry_factory = track_geometry_factory - if geometry_dataset is None: - self._geometry_dataset = dict[ + if geometry_datasets is None: + self._geometry_datasets = dict[ RelativeOffsetCoordinate, TrackGeometryDataset ]() else: - self._geometry_dataset = geometry_dataset + self._geometry_datasets = geometry_datasets @staticmethod def from_list( @@ -198,7 +198,7 @@ def from_dataframe( valid_tracks = classified_tracks.loc[ classified_tracks[track.TRACK_ID].isin(valid_track_ids), : ] - return PandasTrackDataset(valid_tracks, geometry_dataset=geometry_dataset) + return PandasTrackDataset(valid_tracks, geometry_datasets=geometry_dataset) def add_all(self, other: Iterable[Track]) -> TrackDataset: new_tracks = self.__get_tracks(other) @@ -220,7 +220,7 @@ def _add_to_geometry_dataset( self, new_tracks: TrackDataset ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() - for offset, geometries in self._geometry_dataset.items(): + for offset, geometries in self._geometry_datasets.items(): updated[offset] = geometries.add_all(new_tracks) return updated @@ -312,9 +312,9 @@ def _get_geometry_dataset_for( TrackGeometryDataset: the track geometry dataset with the given offset applied. """ - if (geometry_dataset := self._geometry_dataset.get(offset, None)) is None: + if (geometry_dataset := self._geometry_datasets.get(offset, None)) is None: geometry_dataset = self._track_geometry_factory(self, offset) - self._geometry_dataset[offset] = geometry_dataset + self._geometry_datasets[offset] = geometry_dataset return geometry_dataset def intersection_points( diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 33eba02f0..235aacbcc 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -122,7 +122,7 @@ def test_add_all(self) -> None: ) for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) - assert merged._geometry_dataset == {} + assert merged._geometry_datasets == {} def test_add_two_existing_pandas_datasets(self) -> None: first_track = self.__build_track("1") @@ -134,7 +134,7 @@ def test_add_two_existing_pandas_datasets(self) -> None: for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): assert_equal_track_properties(actual, expected) - assert merged._geometry_dataset == {} + assert merged._geometry_datasets == {} def __build_track(self, track_id: str, length: int = 5) -> Track: builder = TrackBuilder() From 776261aefdd7d851f243c609cdc376c917adbbeb Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:40:03 +0100 Subject: [PATCH 054/107] Collect fixtures to be used across several modules and move them to conftest.py --- .../OTAnalytics/plugin_datastore/conftest.py | 77 +++++++++++++++++++ .../test_python_track_storage.py | 70 ----------------- 2 files changed, 77 insertions(+), 70 deletions(-) create mode 100644 tests/OTAnalytics/plugin_datastore/conftest.py diff --git a/tests/OTAnalytics/plugin_datastore/conftest.py b/tests/OTAnalytics/plugin_datastore/conftest.py new file mode 100644 index 000000000..2b6ad9c36 --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/conftest.py @@ -0,0 +1,77 @@ +import pytest + +from OTAnalytics.domain.track import Track +from tests.conftest import TrackBuilder + + +@pytest.fixture +def first_track() -> Track: + track_builder = TrackBuilder() + _class = "car" + + track_builder.add_track_id("1") + track_builder.add_track_class(_class) + track_builder.add_second(1) + track_builder.add_frame(1) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + track_builder.add_track_class(_class) + track_builder.add_second(2) + track_builder.add_frame(2) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def first_track_continuing() -> Track: + track_builder = TrackBuilder() + _class = "truck" + track_builder.add_track_id("1") + track_builder.add_track_class(_class) + track_builder.add_second(3) + track_builder.add_frame(3) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + track_builder.add_track_class(_class) + track_builder.add_second(4) + track_builder.add_frame(4) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + track_builder.add_track_class(_class) + track_builder.add_second(5) + track_builder.add_frame(5) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + return track_builder.build_track() + + +@pytest.fixture +def second_track() -> Track: + track_builder = TrackBuilder() + _class = "pedestrian" + track_builder.add_track_id("2") + track_builder.add_track_class(_class) + track_builder.add_second(1) + track_builder.add_frame(1) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + track_builder.add_track_class(_class) + track_builder.add_second(2) + track_builder.add_frame(2) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + track_builder.add_track_class(_class) + track_builder.add_second(3) + track_builder.add_frame(3) + track_builder.add_detection_class(_class) + track_builder.append_detection() + + return track_builder.build_track() diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index 617ebc284..5a500eb48 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -228,76 +228,6 @@ def test_first_and_last_detection(self, valid_detection: Detection) -> None: class TestPythonTrackDataset: - @pytest.fixture - def first_track(self) -> Track: - track_builder = TrackBuilder() - _class = "car" - - track_builder.add_track_id("1") - track_builder.add_track_class(_class) - track_builder.add_second(1) - track_builder.add_frame(1) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - track_builder.add_track_class(_class) - track_builder.add_second(2) - track_builder.add_frame(2) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - return track_builder.build_track() - - @pytest.fixture - def first_track_continuing(self) -> Track: - track_builder = TrackBuilder() - _class = "truck" - track_builder.add_track_id("1") - track_builder.add_track_class(_class) - track_builder.add_second(3) - track_builder.add_frame(3) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - track_builder.add_track_class(_class) - track_builder.add_second(4) - track_builder.add_frame(4) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - track_builder.add_track_class(_class) - track_builder.add_second(5) - track_builder.add_frame(5) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - return track_builder.build_track() - - @pytest.fixture - def second_track(self) -> Track: - track_builder = TrackBuilder() - _class = "pedestrian" - track_builder.add_track_id("2") - track_builder.add_track_class(_class) - track_builder.add_second(1) - track_builder.add_frame(1) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - track_builder.add_track_class(_class) - track_builder.add_second(2) - track_builder.add_frame(2) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - track_builder.add_track_class(_class) - track_builder.add_second(3) - track_builder.add_frame(3) - track_builder.add_detection_class(_class) - track_builder.append_detection() - - return track_builder.build_track() - @staticmethod def create_track_dataset(size: int) -> PythonTrackDataset: dataset: dict[TrackId, Track] = {} From d36a4bd3f2cbe8a2f2370c6e3bb7af9b9a50ff8f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:41:20 +0100 Subject: [PATCH 055/107] Reuse existing TrackGeometryDataset when adding new tracks --- OTAnalytics/plugin_datastore/python_track_store.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 0e7d2d1cb..bd6ba9394 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -277,7 +277,16 @@ def __merge(self, other: dict[TrackId, Track]) -> TrackDataset: except TrackHasNoDetectionError as build_error: logger().exception(build_error, exc_info=True) merged = self._tracks | merged_tracks - return PythonTrackDataset(merged) + updated_geometry_dataset = self._add_to_geometry_dataset(merged_tracks.values()) + return PythonTrackDataset(merged, updated_geometry_dataset) + + def _add_to_geometry_dataset( + self, new_tracks: Iterable[Track] + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() + for offset, geometries in self._geometry_datasets.items(): + updated[offset] = geometries.add_all(new_tracks) + return updated def _get_existing_detections(self, track_id: TrackId) -> list[Detection]: """ From a16b9451f666987de42dd0d6caabb995514db30e Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:47:10 +0100 Subject: [PATCH 056/107] Do not filter tracks with certain number of tracks when creating a new PandasTrackDataset The filtering by the number of detections should be the sole responsibility of the TrackParser. By using a cutting section it is possible and valid to have tracks less detections than the given limit used by the TrackParser. Thus, no filtering should be applied when creating a new TrackDataset, resulting in missing tracks. --- OTAnalytics/plugin_datastore/track_store.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 2c95c111b..a66064ba0 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -12,7 +12,6 @@ from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - MIN_NUMBER_OF_DETECTIONS, TRACK_GEOMETRY_FACTORY, Detection, IntersectionPoint, @@ -186,19 +185,7 @@ def from_dataframe( if tracks.empty: return PandasTrackDataset() classified_tracks = _assign_track_classification(tracks, calculator) - detections_per_track = ( - classified_tracks.groupby(by=track.TRACK_ID)[track.FRAME] - .count() - .reset_index() - ) - valid_track_ids = detections_per_track.loc[ - detections_per_track[track.FRAME] >= MIN_NUMBER_OF_DETECTIONS, - track.TRACK_ID, - ] - valid_tracks = classified_tracks.loc[ - classified_tracks[track.TRACK_ID].isin(valid_track_ids), : - ] - return PandasTrackDataset(valid_tracks, geometry_datasets=geometry_dataset) + return PandasTrackDataset(classified_tracks, geometry_datasets=geometry_dataset) def add_all(self, other: Iterable[Track]) -> TrackDataset: new_tracks = self.__get_tracks(other) From 19e29560bc2e4688ea628e0d6652bd766dddff64 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:49:51 +0100 Subject: [PATCH 057/107] Extend test cases for adding to a TrackDataset --- .../test_python_track_storage.py | 69 ++++++++++++++----- .../plugin_datastore/test_track_store.py | 65 ++++++++++++++++- 2 files changed, 116 insertions(+), 18 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index 5a500eb48..60a14a45e 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -1,11 +1,19 @@ from datetime import datetime from pathlib import Path +from typing import cast from unittest.mock import Mock import pytest from OTAnalytics.domain.event import VIDEO_NAME -from OTAnalytics.domain.track import Detection, Track, TrackHasNoDetectionError, TrackId +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.track import ( + Detection, + Track, + TrackGeometryDataset, + TrackHasNoDetectionError, + TrackId, +) from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonDetection, @@ -239,31 +247,60 @@ def create_track_dataset(size: int) -> PythonTrackDataset: return PythonTrackDataset(dataset) - def test_add_all(self, first_track: Track, second_track: Track) -> None: + def test_add_all_to_empty(self, first_track: Track, second_track: Track) -> None: tracks = [first_track, second_track] dataset = PythonTrackDataset() - result_dataset = dataset.add_all(tracks) + result_dataset = cast(PythonTrackDataset, dataset.add_all(tracks)) assert list(result_dataset) == tracks + assert result_dataset._geometry_datasets == {} def test_add_all_merge_tracks( - self, first_track: Track, first_track_continuing: Track + self, first_track: Track, first_track_continuing: Track, second_track: Track ) -> None: - dataset = PythonTrackDataset() - dataset_with_first_track = dataset.add_all([first_track]) - assert list(dataset_with_first_track) == [first_track] - - dataset_merged_track = dataset_with_first_track.add_all( - [first_track_continuing] + geometry_dataset_no_offset = Mock(spec=TrackGeometryDataset) + updated_geometry_dataset_no_offset = Mock() + geometry_dataset_no_offset.add_all.return_value = ( + updated_geometry_dataset_no_offset + ) + geometry_dataset_with_offset = Mock(spec=TrackGeometryDataset) + updated_geometry_dataset_with_offset = Mock() + geometry_dataset_with_offset.add_all.return_value = ( + updated_geometry_dataset_with_offset + ) + geometry_datasets = { + RelativeOffsetCoordinate(0, 0): cast( + TrackGeometryDataset, geometry_dataset_no_offset + ), + RelativeOffsetCoordinate(0.5, 0.5): cast( + TrackGeometryDataset, geometry_dataset_with_offset + ), + } + dataset = PythonTrackDataset({first_track.id: first_track}, geometry_datasets) + dataset_merged_track = cast( + PythonTrackDataset, dataset.add_all([first_track_continuing, second_track]) + ) + expected_merged_track = PythonTrack( + first_track.id, + first_track_continuing.classification, + first_track.detections + first_track_continuing.detections, ) - assert list(dataset_merged_track) == [ - PythonTrack( - first_track.id, - first_track_continuing.classification, - first_track.detections + first_track_continuing.detections, - ) + expected_merged_track, + second_track, + ] + assert list(geometry_dataset_no_offset.add_all.call_args_list[0][0][0]) == [ + expected_merged_track, + second_track, + ] + assert list(geometry_dataset_with_offset.add_all.call_args_list[0][0][0]) == [ + expected_merged_track, + second_track, ] + assert dataset_merged_track._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): updated_geometry_dataset_no_offset, + RelativeOffsetCoordinate(0.5, 0.5): updated_geometry_dataset_with_offset, + } def test_add_nothing(self, first_track: Track) -> None: dataset = PythonTrackDataset() diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 235aacbcc..de7cd69b3 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -1,20 +1,27 @@ from typing import cast +from unittest.mock import Mock import pytest from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.track import Track, TrackDataset, TrackId -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.track import Track, TrackDataset, TrackGeometryDataset, TrackId +from OTAnalytics.plugin_datastore.python_track_store import ( + PythonTrack, + PythonTrackDataset, +) from OTAnalytics.plugin_datastore.track_store import ( PandasDetection, PandasTrack, PandasTrackDataset, + _convert_tracks, ) from tests.conftest import ( TrackBuilder, assert_equal_detection_properties, assert_equal_track_properties, + assert_track_datasets_equal, ) @@ -136,6 +143,60 @@ def test_add_two_existing_pandas_datasets(self) -> None: assert_equal_track_properties(actual, expected) assert merged._geometry_datasets == {} + def test_add_all_merge_tracks( + self, first_track: Track, first_track_continuing: Track, second_track: Track + ) -> None: + geometry_dataset_no_offset = Mock(spec=TrackGeometryDataset) + updated_geometry_dataset_no_offset = Mock() + geometry_dataset_no_offset.add_all.return_value = ( + updated_geometry_dataset_no_offset + ) + geometry_dataset_with_offset = Mock(spec=TrackGeometryDataset) + updated_geometry_dataset_with_offset = Mock() + geometry_dataset_with_offset.add_all.return_value = ( + updated_geometry_dataset_with_offset + ) + geometry_datasets = { + RelativeOffsetCoordinate(0, 0): cast( + TrackGeometryDataset, geometry_dataset_no_offset + ), + RelativeOffsetCoordinate(0.5, 0.5): cast( + TrackGeometryDataset, geometry_dataset_with_offset + ), + } + dataset = PandasTrackDataset.from_dataframe( + _convert_tracks([first_track]), geometry_datasets + ) + dataset_merged_track = cast( + PandasTrackDataset, dataset.add_all([first_track_continuing, second_track]) + ) + expected_merged_track = PythonTrack( + first_track.id, + first_track_continuing.classification, + first_track.detections + first_track_continuing.detections, + ) + expected_dataset = PandasTrackDataset.from_list( + [expected_merged_track, second_track] + ) + assert_track_datasets_equal(dataset_merged_track, expected_dataset) + + for actual_track, expected_track in zip( + geometry_dataset_no_offset.add_all.call_args_list[0][0][0], + expected_dataset, + ): + assert_equal_track_properties(actual_track, expected_track) + + for actual_track, expected_track in zip( + geometry_dataset_with_offset.add_all.call_args_list[0][0][0], + expected_dataset, + ): + assert_equal_track_properties(actual_track, expected_track) + + assert dataset_merged_track._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): updated_geometry_dataset_no_offset, + RelativeOffsetCoordinate(0.5, 0.5): updated_geometry_dataset_with_offset, + } + def __build_track(self, track_id: str, length: int = 5) -> Track: builder = TrackBuilder() builder.add_track_id(track_id) From 22a58e503d60073415076447532edb5821d15d28 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 30 Nov 2023 21:04:20 +0100 Subject: [PATCH 058/107] Refactor code duplication --- .../OTAnalytics/plugin_datastore/conftest.py | 24 ++++++++++- .../test_python_track_storage.py | 36 ++++++++-------- .../plugin_datastore/test_track_store.py | 42 ++++++++----------- 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/conftest.py b/tests/OTAnalytics/plugin_datastore/conftest.py index 2b6ad9c36..e12218809 100644 --- a/tests/OTAnalytics/plugin_datastore/conftest.py +++ b/tests/OTAnalytics/plugin_datastore/conftest.py @@ -1,7 +1,27 @@ +from typing import Iterable +from unittest.mock import Mock + import pytest -from OTAnalytics.domain.track import Track -from tests.conftest import TrackBuilder +from OTAnalytics.domain.track import Track, TrackGeometryDataset +from tests.conftest import TrackBuilder, assert_equal_track_properties + + +def create_mock_geometry_dataset() -> tuple[Mock, Mock]: + geometry_dataset = Mock(spec=TrackGeometryDataset) + updated_geometry_dataset = Mock() + geometry_dataset.add_all.return_value = updated_geometry_dataset + geometry_dataset.remove.return_value = updated_geometry_dataset + return geometry_dataset, updated_geometry_dataset + + +def assert_track_geometry_dataset_add_all_called_correctly( + called_method: Mock, expected_arg: Iterable[Track] +) -> None: + for actual_track, expected_track in zip( + called_method.call_args_list[0][0][0], expected_arg + ): + assert_equal_track_properties(actual_track, expected_track) @pytest.fixture diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index 60a14a45e..74199d734 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -22,6 +22,10 @@ ) from OTAnalytics.plugin_parser import ottrk_dataformat as ottrk_format from tests.conftest import TrackBuilder +from tests.OTAnalytics.plugin_datastore.conftest import ( + assert_track_geometry_dataset_add_all_called_correctly, + create_mock_geometry_dataset, +) @pytest.fixture @@ -258,16 +262,14 @@ def test_add_all_to_empty(self, first_track: Track, second_track: Track) -> None def test_add_all_merge_tracks( self, first_track: Track, first_track_continuing: Track, second_track: Track ) -> None: - geometry_dataset_no_offset = Mock(spec=TrackGeometryDataset) - updated_geometry_dataset_no_offset = Mock() - geometry_dataset_no_offset.add_all.return_value = ( - updated_geometry_dataset_no_offset - ) - geometry_dataset_with_offset = Mock(spec=TrackGeometryDataset) - updated_geometry_dataset_with_offset = Mock() - geometry_dataset_with_offset.add_all.return_value = ( - updated_geometry_dataset_with_offset - ) + ( + geometry_dataset_no_offset, + updated_geometry_dataset_no_offset, + ) = create_mock_geometry_dataset() + ( + geometry_dataset_with_offset, + updated_geometry_dataset_with_offset, + ) = create_mock_geometry_dataset() geometry_datasets = { RelativeOffsetCoordinate(0, 0): cast( TrackGeometryDataset, geometry_dataset_no_offset @@ -289,14 +291,12 @@ def test_add_all_merge_tracks( expected_merged_track, second_track, ] - assert list(geometry_dataset_no_offset.add_all.call_args_list[0][0][0]) == [ - expected_merged_track, - second_track, - ] - assert list(geometry_dataset_with_offset.add_all.call_args_list[0][0][0]) == [ - expected_merged_track, - second_track, - ] + assert_track_geometry_dataset_add_all_called_correctly( + geometry_dataset_no_offset.add_all, [expected_merged_track, second_track] + ) + assert_track_geometry_dataset_add_all_called_correctly( + geometry_dataset_with_offset.add_all, [expected_merged_track, second_track] + ) assert dataset_merged_track._geometry_datasets == { RelativeOffsetCoordinate(0, 0): updated_geometry_dataset_no_offset, RelativeOffsetCoordinate(0.5, 0.5): updated_geometry_dataset_with_offset, diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index de7cd69b3..1b13488c3 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -1,5 +1,4 @@ from typing import cast -from unittest.mock import Mock import pytest from pandas import DataFrame, Series @@ -23,6 +22,10 @@ assert_equal_track_properties, assert_track_datasets_equal, ) +from tests.OTAnalytics.plugin_datastore.conftest import ( + assert_track_geometry_dataset_add_all_called_correctly, + create_mock_geometry_dataset, +) class TestPandasDetection: @@ -146,16 +149,14 @@ def test_add_two_existing_pandas_datasets(self) -> None: def test_add_all_merge_tracks( self, first_track: Track, first_track_continuing: Track, second_track: Track ) -> None: - geometry_dataset_no_offset = Mock(spec=TrackGeometryDataset) - updated_geometry_dataset_no_offset = Mock() - geometry_dataset_no_offset.add_all.return_value = ( - updated_geometry_dataset_no_offset - ) - geometry_dataset_with_offset = Mock(spec=TrackGeometryDataset) - updated_geometry_dataset_with_offset = Mock() - geometry_dataset_with_offset.add_all.return_value = ( - updated_geometry_dataset_with_offset - ) + ( + geometry_dataset_no_offset, + updated_geometry_dataset_no_offset, + ) = create_mock_geometry_dataset() + ( + geometry_dataset_with_offset, + updated_geometry_dataset_with_offset, + ) = create_mock_geometry_dataset() geometry_datasets = { RelativeOffsetCoordinate(0, 0): cast( TrackGeometryDataset, geometry_dataset_no_offset @@ -179,19 +180,12 @@ def test_add_all_merge_tracks( [expected_merged_track, second_track] ) assert_track_datasets_equal(dataset_merged_track, expected_dataset) - - for actual_track, expected_track in zip( - geometry_dataset_no_offset.add_all.call_args_list[0][0][0], - expected_dataset, - ): - assert_equal_track_properties(actual_track, expected_track) - - for actual_track, expected_track in zip( - geometry_dataset_with_offset.add_all.call_args_list[0][0][0], - expected_dataset, - ): - assert_equal_track_properties(actual_track, expected_track) - + assert_track_geometry_dataset_add_all_called_correctly( + geometry_dataset_no_offset.add_all, expected_dataset + ) + assert_track_geometry_dataset_add_all_called_correctly( + geometry_dataset_with_offset.add_all, expected_dataset + ) assert dataset_merged_track._geometry_datasets == { RelativeOffsetCoordinate(0, 0): updated_geometry_dataset_no_offset, RelativeOffsetCoordinate(0.5, 0.5): updated_geometry_dataset_with_offset, From 591338f8a049a192a0bbf872bff5abe680cf3bd5 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:25:41 +0100 Subject: [PATCH 059/107] Reuse existing geometry datasets when removing tracks from TrackDataset --- OTAnalytics/plugin_datastore/python_track_store.py | 11 ++++++++++- OTAnalytics/plugin_datastore/track_store.py | 11 ++++++++++- .../plugin_datastore/test_python_track_storage.py | 14 ++++++++++---- .../plugin_datastore/test_track_store.py | 12 ++++++++++-- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index bd6ba9394..7ee5868fc 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -312,7 +312,16 @@ def get_for(self, id: TrackId) -> Optional[Track]: def remove(self, track_id: TrackId) -> TrackDataset: new_tracks = self._tracks.copy() del new_tracks[track_id] - return PythonTrackDataset(new_tracks) + updated_geometry_datasets = self._remove_from_geometry_datasets({track_id}) + return PythonTrackDataset(new_tracks, updated_geometry_datasets) + + def _remove_from_geometry_datasets( + self, track_ids: set[TrackId] + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + updated = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + updated[offset] = geometry_dataset.remove(track_ids) + return updated def clear(self) -> TrackDataset: return PythonTrackDataset() diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index a66064ba0..c1b8c3b8c 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -231,7 +231,16 @@ def remove(self, track_id: TrackId) -> "TrackDataset": remaining_tracks = self._dataset.loc[ self._dataset[track.TRACK_ID] != track_id.id, : ] - return PandasTrackDataset(remaining_tracks.copy()) + updated_geometry_datasets = self._remove_from_geometry_dataset({track_id}) + return PandasTrackDataset(remaining_tracks.copy(), updated_geometry_datasets) + + def _remove_from_geometry_dataset( + self, track_ids: set[TrackId] + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + updated_dataset = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + updated_dataset[offset] = geometry_dataset.remove(track_ids) + return updated_dataset def as_list(self) -> list[Track]: if self._dataset.empty: diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index 74199d734..5c1f54ee0 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -330,11 +330,17 @@ def test_clear(self, first_track: Track, second_track: Track) -> None: assert list(result) == [] def test_remove(self, first_track: Track, second_track: Track) -> None: - dataset = PythonTrackDataset() - result_dataset = dataset.add_all([first_track, second_track]) - - result = result_dataset.remove(second_track.id) + geometry_dataset, updated_geometry_dataset = create_mock_geometry_dataset() + dataset = PythonTrackDataset( + {first_track.id: first_track, second_track.id: second_track}, + {RelativeOffsetCoordinate(0, 0): geometry_dataset}, + ) + result = cast(PythonTrackDataset, dataset.remove(second_track.id)) assert list(result) == [first_track] + assert result._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): updated_geometry_dataset + } + geometry_dataset.remove.assert_called_once_with({second_track.id}) @pytest.mark.parametrize( "num_tracks,batches,expected_batches", [(10, 1, 1), (10, 4, 4), (3, 4, 3)] diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 1b13488c3..4b8634a7a 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -227,14 +227,22 @@ def test_clear(self) -> None: def test_remove(self) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - dataset = PandasTrackDataset.from_list([first_track, second_track]) + tracks_df = _convert_tracks([first_track, second_track]) + geometry_dataset, updated_geometry_dataset = create_mock_geometry_dataset() + dataset = PandasTrackDataset.from_dataframe( + tracks_df, {RelativeOffsetCoordinate(0, 0): geometry_dataset} + ) - removed_track_set = dataset.remove(first_track.id) + removed_track_set = cast(PandasTrackDataset, dataset.remove(first_track.id)) for actual, expected in zip( removed_track_set.as_list(), PandasTrackDataset.from_list([second_track]).as_list(), ): assert_equal_track_properties(actual, expected) + geometry_dataset.remove.assert_called_once_with({first_track.id}) + assert removed_track_set._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): updated_geometry_dataset, + } def test_len(self) -> None: first_track = self.__build_track("1") From 84b6bc980574be1896b3324df0bc61647bf576fa Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:29:03 +0100 Subject: [PATCH 060/107] Fix broken dependencies in main application --- OTAnalytics/plugin_ui/main_application.py | 17 +++++++++-------- .../plugin_ui/visualization/visualization.py | 5 ++++- tests/benchmark_otanalytics.py | 14 ++++++++------ 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index b95c32473..6312bf344 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -291,6 +291,7 @@ def start_gui(self) -> None: create_events = self._create_use_case_create_events( section_repository, clear_all_events, + get_all_tracks, get_tracks_without_single_detections, add_events, DEFAULT_NUM_PROCESSES, @@ -298,7 +299,7 @@ def start_gui(self) -> None: intersect_tracks_with_sections = ( self._create_use_case_create_intersection_events( section_repository, - get_tracks_without_single_detections, + get_all_tracks, add_events, DEFAULT_NUM_PROCESSES, ) @@ -459,6 +460,7 @@ def start_cli(self, cli_args: CliArguments) -> None: create_events = self._create_use_case_create_events( section_repository, clear_all_events, + get_all_tracks, get_tracks_without_single_detections, add_events, cli_args.num_processes, @@ -640,7 +642,7 @@ def _create_flow_generator( def _create_use_case_create_intersection_events( self, section_repository: SectionRepository, - get_tracks: GetTracksWithoutSingleDetections, + get_tracks: GetAllTracks, add_events: AddEvents, num_processes: int, ) -> CreateIntersectionEvents: @@ -648,9 +650,7 @@ def _create_use_case_create_intersection_events( return SimpleCreateIntersectionEvents(intersect, section_repository, add_events) @staticmethod - def _create_intersect( - get_tracks: GetTracksWithoutSingleDetections, num_processes: int - ) -> RunIntersect: + def _create_intersect(get_tracks: GetAllTracks, num_processes: int) -> RunIntersect: return ShapelyRunIntersect( intersect_parallelizer=MultiprocessingIntersectParallelization( num_processes @@ -693,17 +693,18 @@ def _create_use_case_create_events( self, section_repository: SectionRepository, clear_events: ClearAllEvents, - get_tracks: GetTracksWithoutSingleDetections, + get_all_tracks: GetAllTracks, + get_all_tracks_without_single_detections: GetTracksWithoutSingleDetections, add_events: AddEvents, num_processes: int, ) -> CreateEvents: - run_intersect = self._create_intersect(get_tracks, num_processes) + run_intersect = self._create_intersect(get_all_tracks, num_processes) create_intersection_events = SimpleCreateIntersectionEvents( run_intersect, section_repository, add_events ) scene_action_detector = SceneActionDetector(SceneEventBuilder()) create_scene_events = SimpleCreateSceneEvents( - get_tracks, scene_action_detector, add_events + get_all_tracks_without_single_detections, scene_action_detector, add_events ) return CreateEvents( clear_events, create_intersection_events, create_scene_events diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 45deb611f..224c3e12d 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -322,7 +322,10 @@ def _create_pandas_track_provider( ) -> PandasTrackProvider: dataframe_filter_builder = self._create_dataframe_filter_builder() # return PandasTrackProvider( - # datastore, self._track_view_state, dataframe_filter_builder, progressbar + # self._track_repository, + # self._track_view_state, + # dataframe_filter_builder, + # progressbar, # ) return CachedPandasTrackProvider( self._track_repository, diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index 46549e192..271e8c95a 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -14,6 +14,7 @@ from OTAnalytics.application.use_cases.event_repository import AddEvents, ClearAllEvents from OTAnalytics.application.use_cases.section_repository import GetSectionsById from OTAnalytics.application.use_cases.track_repository import ( + GetAllTracks, GetTracksWithoutSingleDetections, ) from OTAnalytics.domain.event import EventRepository @@ -28,7 +29,6 @@ PandasByMaxConfidence, PandasTrackDataset, ) -from OTAnalytics.plugin_intersect.shapely.intersect import ShapelyIntersector from OTAnalytics.plugin_parser.otvision_parser import ( OtFlowParser, OttrkParser, @@ -68,10 +68,8 @@ def _build_tracks_intersecting_sections( track_repository: TrackRepository, ) -> TracksIntersectingSections: starter = ApplicationStarter() - get_all_tracks = GetTracksWithoutSingleDetections(track_repository) - return starter._create_tracks_intersecting_sections( - get_all_tracks, ShapelyIntersector() - ) + get_all_tracks = GetAllTracks(track_repository) + return starter._create_tracks_intersecting_sections(get_all_tracks) def _build_create_events( @@ -81,12 +79,16 @@ def _build_create_events( ) -> CreateEvents: starter = ApplicationStarter() clear_all_events = ClearAllEvents(event_repository) - get_tracks = GetTracksWithoutSingleDetections(track_repository) + get_tracks_without_single_detections = GetTracksWithoutSingleDetections( + track_repository + ) + get_tracks = GetAllTracks(track_repository) add_events = AddEvents(event_repository) create_events = starter._create_use_case_create_events( section_repository, clear_all_events, get_tracks, + get_tracks_without_single_detections, add_events, num_processes=NUM_PROCESSES, ) From caf54b27a9f3c9607a8b9bde7f22b30249f89f1f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:26:25 +0100 Subject: [PATCH 061/107] Add offset property to TrackGeometryDataset --- OTAnalytics/domain/track.py | 5 +++++ .../plugin_datastore/track_geometry_store/pygeos_store.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 3e0933403..bc6132cef 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -699,6 +699,11 @@ def track_ids(self) -> set[str]: """ raise NotImplementedError + @property + @abstractmethod + def offset(self) -> RelativeOffsetCoordinate: + raise NotImplementedError + @staticmethod @abstractmethod def from_track_dataset( diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 67dfa640b..4b6a85fd4 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -114,6 +114,10 @@ def _create_empty(self) -> DataFrame: def track_ids(self) -> set[str]: return set(self._dataset.index) + @property + def offset(self) -> RelativeOffsetCoordinate: + return self._offset + @property def empty(self) -> bool: """Whether dataset is empty. From 70e9525cb569166fc91b813f868fe38f5f21f13a Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:27:02 +0100 Subject: [PATCH 062/107] Get track geometries only for a set of track ids --- OTAnalytics/domain/track.py | 14 +++++++++++++ .../track_geometry_store/pygeos_store.py | 6 ++++++ .../track_geometry_store/test_pygeos_store.py | 21 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index bc6132cef..d6093a141 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -738,6 +738,20 @@ def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": """ raise NotImplementedError + @abstractmethod + def get_for(self, track_ids: Iterable[str]) -> "TrackGeometryDataset": + """Get geometries for given track ids if they exist. + + Ids that do not exist will not be included in the dataset. + + Args: + track_ids (Iterable[str]): the track ids. + + Returns: + TrackGeometryDataset: the dataset with tracks. + """ + raise NotImplementedError + @abstractmethod def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 4b6a85fd4..9af791d86 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -194,6 +194,12 @@ def remove(self, ids: Iterable[TrackId]) -> TrackGeometryDataset: ) return PygeosTrackGeometryDataset(self._offset, updated) + def get_for(self, track_ids: Iterable[str]) -> "TrackGeometryDataset": + _ids = self._dataset.index.intersection(track_ids) + + filtered_df = self._dataset.loc[_ids] + return PygeosTrackGeometryDataset(self.offset, filtered_df) + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: intersecting_tracks = set() section_geoms = line_sections_to_pygeos_multi(sections) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 31a2743dd..cfae74add 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -311,6 +311,7 @@ def assert_track_geometry_dataset_equals( ) -> None: assert isinstance(to_compare, PygeosTrackGeometryDataset) assert isinstance(other, PygeosTrackGeometryDataset) + assert to_compare.offset == other.offset assert to_compare._dataset.equals(other._dataset) # noqa @@ -619,6 +620,26 @@ def test_contained_by_sections( ) assert result == contained_by_section_test_case.expected_result + def test_get_for_existing(self, first_track: Track, second_track: Track) -> None: + track_dataset = create_track_dataset([first_track, second_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) + result = geometry_dataset.get_for([first_track.id.id]) + expected = create_geometry_dataset_from([first_track], BASE_GEOMETRY) + assert_track_geometry_dataset_equals(result, expected) + + def test_get_for_not_existing( + self, first_track: Track, second_track: Track + ) -> None: + track_dataset = create_track_dataset([first_track, second_track]) + geometry_dataset = PygeosTrackGeometryDataset.from_track_dataset( + track_dataset, BASE_GEOMETRY + ) + result = geometry_dataset.get_for(["not-existing-track"]) + expected = create_geometry_dataset_from([], BASE_GEOMETRY) + assert_track_geometry_dataset_equals(result, expected) + class TestProfiling: ROUNDS = 1 From 6874841be7d4ec04d3b3170354da648aae7e1754 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:07:46 +0100 Subject: [PATCH 063/107] Use factory method when creating PandasTrackDataset --- OTAnalytics/plugin_datastore/track_store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index c1b8c3b8c..933aba1cc 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -232,7 +232,9 @@ def remove(self, track_id: TrackId) -> "TrackDataset": self._dataset[track.TRACK_ID] != track_id.id, : ] updated_geometry_datasets = self._remove_from_geometry_dataset({track_id}) - return PandasTrackDataset(remaining_tracks.copy(), updated_geometry_datasets) + return PandasTrackDataset.from_dataframe( + remaining_tracks.copy(), updated_geometry_datasets + ) def _remove_from_geometry_dataset( self, track_ids: set[TrackId] From 7f154eb4013773fad5c4c2e22a1678dc648222d1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:08:26 +0100 Subject: [PATCH 064/107] Reuse existing geometries when splitting TrackDatasets --- .../plugin_datastore/python_track_store.py | 25 ++++++++-- OTAnalytics/plugin_datastore/track_store.py | 13 ++++- .../OTAnalytics/plugin_datastore/conftest.py | 6 ++- .../test_python_track_storage.py | 47 ++++++++++++++++++- .../plugin_datastore/test_track_store.py | 45 ++++++++++++++++++ 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 7ee5868fc..741adb078 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -333,10 +333,27 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: dataset_size = len(self._tracks) batch_size = ceil(dataset_size / batches) - return [ - PythonTrackDataset(dict(batch), calculator=self._calculator) - for batch in batched(self._tracks.items(), batch_size) - ] + dataset_batches = [] + for batch in batched(self._tracks.items(), batch_size): + current_batch = dict(batch) + current_geometry_datasets = self._get_geometries_for(current_batch.keys()) + dataset_batches.append( + PythonTrackDataset( + current_batch, + current_geometry_datasets, + calculator=self._calculator, + ) + ) + return dataset_batches + + def _get_geometries_for( + self, track_ids: Iterable[TrackId] + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + _ids = [track_id.id for track_id in track_ids] + geometry_datasets = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + geometry_datasets[offset] = geometry_dataset.get_for(_ids) + return geometry_datasets def __len__(self) -> int: return len(self._tracks) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 933aba1cc..ed72b4d20 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -265,11 +265,22 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: new_batches: list["TrackDataset"] = [] for batch_ids in batched(all_ids, batch_size): batch_dataset = self._dataset[self._dataset[track.TRACK_ID].isin(batch_ids)] + batch_geometries = self._get_geometries_for(batch_ids) new_batches.append( - PandasTrackDataset(batch_dataset, calculator=self._calculator) + PandasTrackDataset.from_dataframe( + batch_dataset, batch_geometries, calculator=self._calculator + ) ) return new_batches + def _get_geometries_for( + self, track_ids: Iterable[str] + ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: + geometry_datasets = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + geometry_datasets[offset] = geometry_dataset.get_for(track_ids) + return geometry_datasets + def __len__(self) -> int: if self._dataset.empty: return 0 diff --git a/tests/OTAnalytics/plugin_datastore/conftest.py b/tests/OTAnalytics/plugin_datastore/conftest.py index e12218809..58092f812 100644 --- a/tests/OTAnalytics/plugin_datastore/conftest.py +++ b/tests/OTAnalytics/plugin_datastore/conftest.py @@ -7,11 +7,15 @@ from tests.conftest import TrackBuilder, assert_equal_track_properties -def create_mock_geometry_dataset() -> tuple[Mock, Mock]: +def create_mock_geometry_dataset( + get_for_side_effect: list[Mock] | None = None, +) -> tuple[Mock, Mock]: geometry_dataset = Mock(spec=TrackGeometryDataset) updated_geometry_dataset = Mock() geometry_dataset.add_all.return_value = updated_geometry_dataset geometry_dataset.remove.return_value = updated_geometry_dataset + if get_for_side_effect is not None: + geometry_dataset.get_for.side_effect = get_for_side_effect return geometry_dataset, updated_geometry_dataset diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index 5c1f54ee0..dcd6a89fb 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -1,7 +1,7 @@ from datetime import datetime from pathlib import Path from typing import cast -from unittest.mock import Mock +from unittest.mock import Mock, call import pytest @@ -359,6 +359,51 @@ def test_split(self, num_tracks: int, batches: int, expected_batches: int) -> No expected_track = next(it) assert track == expected_track + def test_split_with_existing_geometries( + self, first_track: Track, second_track: Track + ) -> None: + first_batch_geometries_no_offset = Mock() + second_batch_geometries_no_offset = Mock() + geometry_dataset_no_offset, _ = create_mock_geometry_dataset( + [first_batch_geometries_no_offset, second_batch_geometries_no_offset] + ) + first_batch_geometries_with_offset = Mock() + second_batch_geometries_with_offset = Mock() + geometry_dataset_with_offset, _ = create_mock_geometry_dataset( + [first_batch_geometries_with_offset, second_batch_geometries_with_offset] + ) + + geometry_datasets = { + RelativeOffsetCoordinate(0, 0): cast( + TrackGeometryDataset, geometry_dataset_no_offset + ), + RelativeOffsetCoordinate(0.5, 0.5): cast( + TrackGeometryDataset, geometry_dataset_with_offset + ), + } + dataset = PythonTrackDataset( + {first_track.id: first_track, second_track.id: second_track}, + geometry_datasets, + ) + result = cast(list[PythonTrackDataset], dataset.split(batches=2)) + + assert result[0]._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): first_batch_geometries_no_offset, + RelativeOffsetCoordinate(0.5, 0.5): first_batch_geometries_with_offset, + } + assert result[1]._geometry_datasets == { + RelativeOffsetCoordinate(0, 0): second_batch_geometries_no_offset, + RelativeOffsetCoordinate(0.5, 0.5): second_batch_geometries_with_offset, + } + assert geometry_dataset_no_offset.get_for.call_args_list == [ + call([first_track.id.id]), + call([second_track.id.id]), + ] + assert geometry_dataset_with_offset.get_for.call_args_list == [ + call([first_track.id.id]), + call([second_track.id.id]), + ] + def test_filter_by_minimum_detection_length( self, first_track: Track, second_track: Track ) -> None: diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 4b8634a7a..ddbe8b68c 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -1,4 +1,5 @@ from typing import cast +from unittest.mock import Mock, call import pytest from pandas import DataFrame, Series @@ -276,6 +277,50 @@ def test_split(self, num_tracks: int, batches: int, expected_batches: int) -> No ): assert_equal_detection_properties(detection, expected_detection) + def test_split_with_existing_geometries( + self, first_track: Track, second_track: Track + ) -> None: + first_track = self.__build_track("1") + second_track = self.__build_track("2") + first_batch_geometries_no_offset = Mock() + second_batch_geometries_no_offset = Mock() + geometry_dataset_no_offset, _ = create_mock_geometry_dataset( + [first_batch_geometries_no_offset, second_batch_geometries_no_offset] + ) + + first_batch_geometries_with_offset = Mock() + second_batch_geometries_with_offset = Mock() + geometry_dataset_with_offset, _ = create_mock_geometry_dataset( + [first_batch_geometries_with_offset, second_batch_geometries_with_offset] + ) + + geometry_datasets = { + RelativeOffsetCoordinate(0, 0): cast( + TrackGeometryDataset, geometry_dataset_no_offset + ), + RelativeOffsetCoordinate(0.5, 0.5): cast( + TrackGeometryDataset, geometry_dataset_with_offset + ), + } + tracks_df = _convert_tracks([first_track, second_track]) + + dataset = PandasTrackDataset(tracks_df, geometry_datasets) + result = cast(list[PythonTrackDataset], dataset.split(batches=2)) + assert_track_datasets_equal( + result[0], PandasTrackDataset.from_list([first_track]) + ) + assert_track_datasets_equal( + result[1], PandasTrackDataset.from_list([second_track]) + ) + assert geometry_dataset_no_offset.get_for.call_args_list == [ + call((first_track.id.id,)), + call((second_track.id.id,)), + ] + assert geometry_dataset_with_offset.get_for.call_args_list == [ + call((first_track.id.id,)), + call((second_track.id.id,)), + ] + def test_filter_by_minimum_detection_length(self) -> None: first_track = self.__build_track("1", length=5) second_track = self.__build_track("2", length=10) From e7b343b80132a59679014274be1c91287bad4c6f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:56:04 +0100 Subject: [PATCH 065/107] Use multi-index over TrackId and occurrence to speed up loading track files as PandasTrackDatasets --- OTAnalytics/plugin_datastore/track_store.py | 91 ++++++++++--------- OTAnalytics/plugin_parser/pandas_parser.py | 11 ++- .../use_cases/test_track_repository.py | 5 +- .../plugin_datastore/test_track_store.py | 16 ++-- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index ed72b4d20..5f848749f 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -26,7 +26,9 @@ class PandasDetection(Detection): - def __init__(self, data: Series): + def __init__(self, track_id: str, data: Series): + self._track_id = track_id + self._occurrence: Any = data.name self._data = data @property @@ -62,7 +64,7 @@ def frame(self) -> int: @property def occurrence(self) -> datetime: - return self.__get_attribute(track.OCCURRENCE) + return self._occurrence @property def interpolated_detection(self) -> bool: @@ -70,7 +72,7 @@ def interpolated_detection(self) -> bool: @property def track_id(self) -> TrackId: - return TrackId(self.__get_attribute(track.TRACK_ID)) + return TrackId(self._track_id) @property def video_name(self) -> str: @@ -79,11 +81,12 @@ def video_name(self) -> str: @dataclass class PandasTrack(Track): + _id: str _data: DataFrame @property def id(self) -> TrackId: - return TrackId(self._data[track.TRACK_ID].iloc[0]) + return TrackId(self._id) @property def classification(self) -> str: @@ -91,15 +94,15 @@ def classification(self) -> str: @property def detections(self) -> list[Detection]: - return [PandasDetection(row) for index, row in self._data.iterrows()] + return [PandasDetection(self._id, row) for _, row in self._data.iterrows()] @property def first_detection(self) -> Detection: - return PandasDetection(self._data.iloc[0]) + return PandasDetection(self._id, self._data.iloc[0]) @property def last_detection(self) -> Detection: - return PandasDetection(self._data.iloc[-1]) + return PandasDetection(self._id, self._data.iloc[-1]) class PandasTrackClassificationCalculator(ABC): @@ -128,21 +131,24 @@ def calculate(self, detections: DataFrame) -> DataFrame: if detections.empty: return DataFrame() classifications = ( - detections.loc[:, [track.TRACK_ID, track.CLASSIFICATION, track.CONFIDENCE]] + detections.loc[:, [track.CLASSIFICATION, track.CONFIDENCE]] .groupby(by=[track.TRACK_ID, track.CLASSIFICATION]) .sum() .sort_values(track.CONFIDENCE) .groupby(level=0) .tail(1) ) - reset = classifications.reset_index() + reset = classifications.reset_index(level=[1]) renamed = reset.rename( columns={track.CLASSIFICATION: track.TRACK_CLASSIFICATION} ) - return renamed.loc[:, [track.TRACK_ID, track.TRACK_CLASSIFICATION]] + return renamed.loc[:, [track.TRACK_CLASSIFICATION]] DEFAULT_CLASSIFICATOR = PandasByMaxConfidence() +INDEX_NAMES = [track.TRACK_ID, track.OCCURRENCE] +LEVEL_TRACK_ID = 0 +LEVEL_OCCURRENCE = 1 class PandasTrackDataset(TrackDataset): @@ -195,13 +201,15 @@ def add_all(self, other: Iterable[Track]) -> TrackDataset: return PandasTrackDataset.from_dataframe( new_tracks, calculator=self._calculator ) - new_dataset = pandas.concat([self._dataset, new_tracks]) - new_track_ids = new_tracks[track.TRACK_ID].unique() - new_tracks_df = new_dataset[new_dataset[track.TRACK_ID].isin(new_track_ids)] + updated_dataset = pandas.concat([self._dataset, new_tracks]).sort_index() + new_track_ids = new_tracks.index.levels[LEVEL_TRACK_ID] + new_dataset = updated_dataset.loc[new_track_ids] updated_geometry_dataset = self._add_to_geometry_dataset( - PandasTrackDataset.from_dataframe(new_tracks_df) + PandasTrackDataset.from_dataframe(new_dataset) + ) + return PandasTrackDataset.from_dataframe( + updated_dataset, updated_geometry_dataset ) - return PandasTrackDataset.from_dataframe(new_dataset, updated_geometry_dataset) def _add_to_geometry_dataset( self, new_tracks: TrackDataset @@ -228,12 +236,10 @@ def clear(self) -> "TrackDataset": return PandasTrackDataset() def remove(self, track_id: TrackId) -> "TrackDataset": - remaining_tracks = self._dataset.loc[ - self._dataset[track.TRACK_ID] != track_id.id, : - ] + remaining_tracks = self._dataset.drop(track_id.id, errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset({track_id}) return PandasTrackDataset.from_dataframe( - remaining_tracks.copy(), updated_geometry_datasets + remaining_tracks, updated_geometry_datasets ) def _remove_from_geometry_dataset( @@ -247,24 +253,23 @@ def _remove_from_geometry_dataset( def as_list(self) -> list[Track]: if self._dataset.empty: return [] - track_ids = self._dataset.loc[:, track.TRACK_ID].unique() + track_ids = self._get_track_ids() return [self.__create_track_flyweight(current) for current in track_ids] def __create_track_flyweight(self, track_id: str) -> Track: - track_frame = self._dataset.loc[self._dataset[track.TRACK_ID] == track_id, :] - return PandasTrack(track_frame) + track_frame = self._dataset.loc[track_id, :] + return PandasTrack(track_id, track_frame) def as_dataframe(self) -> DataFrame: return self._dataset def split(self, batches: int) -> Sequence["TrackDataset"]: - all_ids = self._dataset[track.TRACK_ID].unique() - dataset_size = len(all_ids) + dataset_size = len(self) batch_size = ceil(dataset_size / batches) new_batches: list["TrackDataset"] = [] - for batch_ids in batched(all_ids, batch_size): - batch_dataset = self._dataset[self._dataset[track.TRACK_ID].isin(batch_ids)] + for batch_ids in batched(self._get_track_ids(), batch_size): + batch_dataset = self._dataset.loc[list(batch_ids), :] batch_geometries = self._get_geometries_for(batch_ids) new_batches.append( PandasTrackDataset.from_dataframe( @@ -273,6 +278,11 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: ) return new_batches + def _get_track_ids(self) -> list[str]: + if self._dataset.empty: + return [] + return list(self._dataset.index.get_level_values(LEVEL_TRACK_ID).unique()) + def _get_geometries_for( self, track_ids: Iterable[str] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: @@ -282,21 +292,14 @@ def _get_geometries_for( return geometry_datasets def __len__(self) -> int: - if self._dataset.empty: - return 0 - return len(self._dataset[track.TRACK_ID].unique()) + return len(self._get_track_ids()) def filter_by_min_detection_length(self, length: int) -> "TrackDataset": - detection_counts_per_track = self._dataset.groupby([track.TRACK_ID])[ - track.CLASSIFICATION - ].count() + detection_counts_per_track = self._dataset.groupby(level=LEVEL_TRACK_ID).size() filtered_ids = detection_counts_per_track[ detection_counts_per_track > length ].index - - filtered_dataset = self._dataset.loc[ - self._dataset[track.TRACK_ID].isin(filtered_ids) - ] + filtered_dataset = self._dataset.loc[filtered_ids] return PandasTrackDataset(filtered_dataset, calculator=self._calculator) def intersecting_tracks( @@ -344,7 +347,7 @@ def _assign_track_classification( ) -> DataFrame: dropped = _drop_track_classification(data) classification_per_track = calculator.calculate(dropped) - return dropped.merge(classification_per_track, on=track.TRACK_ID) + return dropped.merge(classification_per_track, left_index=True, right_index=True) def _drop_track_classification(data: DataFrame) -> DataFrame: @@ -368,12 +371,16 @@ def _convert_tracks(tracks: Iterable[Track]) -> DataFrame: for detection in current_track.detections: prepared.append(detection.to_dict()) - return _sort_tracks(DataFrame(prepared)) + if not prepared: + return DataFrame(prepared) + + df = DataFrame(prepared).set_index(INDEX_NAMES) + df.index.names = INDEX_NAMES + return _sort_tracks(df) def _sort_tracks(track_df: DataFrame) -> DataFrame: - """Sort the given dataframe by trackId and frame, - if both columns are available. + """Sort the given dataframe by trackId and occurrence. Args: track_df (DataFrame): dataframe of tracks @@ -381,6 +388,4 @@ def _sort_tracks(track_df: DataFrame) -> DataFrame: Returns: DataFrame: sorted dataframe by track id and frame """ - if (track.TRACK_ID in track_df.columns) and (track.FRAME in track_df.columns): - track_df.sort_values([track.FRAME], inplace=True) - return track_df + return track_df.sort_index() diff --git a/OTAnalytics/plugin_parser/pandas_parser.py b/OTAnalytics/plugin_parser/pandas_parser.py index 47f5cbc1d..53630d7e2 100644 --- a/OTAnalytics/plugin_parser/pandas_parser.py +++ b/OTAnalytics/plugin_parser/pandas_parser.py @@ -77,12 +77,13 @@ def _parse_as_dataframe( f"Track ids: {too_long_track_ids}" ) - tracks_to_remain = data.loc[ - data[track.TRACK_ID].isin(track_ids_to_remain) - ].copy() - tracks_to_remain.sort_values( - by=[track.TRACK_ID, track.OCCURRENCE], inplace=True + tracks_to_remain = ( + data.loc[data[track.TRACK_ID].isin(track_ids_to_remain)] + .copy() + .set_index([track.TRACK_ID, track.OCCURRENCE]) ) + tracks_to_remain.index.names = [track.TRACK_ID, track.OCCURRENCE] + tracks_to_remain = tracks_to_remain.sort_index() return PandasTrackDataset.from_dataframe( tracks_to_remain, calculator=self._calculator ) diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 407db00dd..e6ae0052d 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -60,16 +60,13 @@ def test_get_all_tracks(self, track_repository: Mock, tracks: TrackDataset) -> N def test_get_as_dataset(self) -> None: expected_dataset = Mock() - dataset = Mock() - dataset.filter_by_min_detection_length.return_value = expected_dataset track_repository = Mock() - track_repository.get_all.return_value = dataset + track_repository.get_all.return_value = expected_dataset get_tracks = GetAllTracks(track_repository) result_dataset = get_tracks.as_dataset() assert result_dataset == expected_dataset track_repository.get_all.assert_called_once() - dataset.filter_by_min_detection_length.assert_called_once_with(2) def test_get_as_list(self) -> None: track_repository = Mock() diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index ddbe8b68c..c380df9e5 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -34,8 +34,11 @@ def test_properties(self) -> None: builder = TrackBuilder() builder.append_detection() python_detection = builder.build_detections()[0] - data = Series(python_detection.to_dict()) - pandas_detection = PandasDetection(data) + data = Series( + python_detection.to_dict(), + name=python_detection.occurrence, + ) + pandas_detection = PandasDetection(python_detection.track_id.id, data) assert_equal_detection_properties(pandas_detection, python_detection) @@ -50,9 +53,10 @@ def test_properties(self) -> None: builder.append_detection() python_track = builder.build_track() detections = [detection.to_dict() for detection in python_track.detections] - data = DataFrame(detections) + data = DataFrame(detections).set_index([track.OCCURRENCE]).sort_index() data[track.TRACK_CLASSIFICATION] = data[track.CLASSIFICATION] - pandas_track = PandasTrack(data) + data = data.drop([track.TRACK_ID], axis=1) + pandas_track = PandasTrack(python_track.id.id, data) assert_equal_track_properties(pandas_track, python_track) @@ -167,10 +171,10 @@ def test_add_all_merge_tracks( ), } dataset = PandasTrackDataset.from_dataframe( - _convert_tracks([first_track]), geometry_datasets + _convert_tracks([first_track_continuing]), geometry_datasets ) dataset_merged_track = cast( - PandasTrackDataset, dataset.add_all([first_track_continuing, second_track]) + PandasTrackDataset, dataset.add_all([first_track, second_track]) ) expected_merged_track = PythonTrack( first_track.id, From fbe5a7faae805e52def691bc85bcd571b139b57c Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:34:57 +0100 Subject: [PATCH 066/107] Disable filtering when trying to get all tracks --- OTAnalytics/application/use_cases/track_repository.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 275ecfcde..9dfaa225f 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -29,8 +29,7 @@ def as_list(self) -> list[Track]: return self.as_dataset().as_list() def as_dataset(self) -> TrackDataset: - tracks = self._track_repository.get_all() - return tracks.filter_by_min_detection_length(2) + return self._track_repository.get_all() class GetTracksWithoutSingleDetections: From 4c7b97affa10baf9bd64879b6a15240e723ce3d6 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:36:03 +0100 Subject: [PATCH 067/107] Fix getting all track ids from PandasTrackDataset --- OTAnalytics/plugin_datastore/track_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 5f848749f..82b598698 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -220,7 +220,7 @@ def _add_to_geometry_dataset( return updated def get_all_ids(self) -> Iterable[TrackId]: - return self._dataset[track.TRACK_ID].apply(lambda track_id: TrackId(track_id)) + return [TrackId(_id) for _id in self._get_track_ids()] def __get_tracks(self, other: Iterable[Track]) -> DataFrame: if isinstance(other, PandasTrackDataset): From 748adcae57b3e38604531f6bb5476e13932e07c8 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:40:21 +0100 Subject: [PATCH 068/107] Fix error caused by new DataFrame format used by PandasTrackDataset Where track ids and occurrence are used as a multi-index. This fix might not be working with a CachedPandasTrackProvider, since no such multi-index is being used there. --- .../plugin_prototypes/track_visualization/track_viz.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index d1104f634..74c506e68 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -290,8 +290,12 @@ def get_data(self) -> DataFrame: data = self._other.get_data() if data.empty: return data - ids: set[str] = {track_id.id for track_id in self._filter.get_ids()} - return data.loc[data[track.TRACK_ID].isin(ids)] + ids = [track_id.id for track_id in self._filter.get_ids()] + # TODO: This only works for DataFrames with track id and occurrence as + # an multi-index. Could not be working with a CachedPandasTrackProvider + # since no such multi-index is being used there. + intersection_of_ids = data.index.get_level_values(0).unique().intersection(ids) + return data.loc[intersection_of_ids] class FilterByClassification(PandasDataFrameProvider): From 8c1cc2525247010cd5fe306d36d5266487d6e21b Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:58:55 +0100 Subject: [PATCH 069/107] Fix filtering DataFrame by occurrence The error is caused by changes in the DataFrame format where track id and occurrence are used as a multi-index. --- OTAnalytics/plugin_filter/dataframe_filter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_filter/dataframe_filter.py b/OTAnalytics/plugin_filter/dataframe_filter.py index b31815a75..682c2f494 100644 --- a/OTAnalytics/plugin_filter/dataframe_filter.py +++ b/OTAnalytics/plugin_filter/dataframe_filter.py @@ -62,6 +62,9 @@ def apply(self, iterable: Iterable[DataFrame]) -> Iterable[DataFrame]: return iterable +INDEX_LEVEL_OCCURRENCE = 1 + + class DataFrameStartsAtOrAfterDate(DataFramePredicate): """Checks if the DataFrame rows start at or after date. @@ -79,7 +82,11 @@ def __init__( self._start_date = start_date def test(self, to_test: DataFrame) -> Series: - return to_test[self.column_name] >= self._start_date + # TODO: Only works for DataFrames that have track id and occurrence as + # multi-index + return ( + to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) >= self._start_date + ) class DataFrameEndsBeforeOrAtDate(DataFramePredicate): @@ -99,7 +106,9 @@ def __init__( self._end_date = end_date def test(self, to_test: DataFrame) -> Series: - return to_test[self.column_name] <= self._end_date + # TODO: Only works for DataFrames that have track id and occurrence as + # multi-index + return to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) <= self._end_date class DataFrameHasClassifications(DataFramePredicate): From 2b8e45dd0d575334116e990e32eb0a86d8a83227 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:04:58 +0100 Subject: [PATCH 070/107] Move create intersection events use case from plugin to application layer --- OTAnalytics/application/analysis/intersect.py | 4 + .../use_cases/create_intersection_events.py | 280 +++++++++++++++++ .../shapely/create_intersection_events.py | 285 +----------------- .../test_create_intersection_events.py | 10 +- tests/OTAnalytics/plugin_ui/test_cli.py | 6 +- 5 files changed, 295 insertions(+), 290 deletions(-) create mode 100644 OTAnalytics/application/use_cases/create_intersection_events.py rename tests/OTAnalytics/{plugin_intersect/shapely => application/use_cases}/test_create_intersection_events.py (99%) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index f404d4108..e7c457715 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -9,6 +9,10 @@ from OTAnalytics.domain.types import EventType +class IntersectionError(Exception): + pass + + class RunIntersect(ABC): """ Interface defining the use case to intersect the given tracks with the given diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py new file mode 100644 index 000000000..73f4ee175 --- /dev/null +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -0,0 +1,280 @@ +from typing import Callable, Iterable + +from OTAnalytics.application.analysis.intersect import ( + IntersectionError, + RunIntersect, + group_sections_by_offset, +) +from OTAnalytics.application.use_cases.track_repository import GetAllTracks +from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder +from OTAnalytics.domain.geometry import ( + DirectionVector2D, + RelativeOffsetCoordinate, + calculate_direction_vector, +) +from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy +from OTAnalytics.domain.section import Area, LineSection, Section +from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.types import EventType + + +class ShapelyIntersectBySmallestTrackSegments(Intersector): + """ + Implements the intersection strategy by splitting up the track in its smallest + segments and intersecting each of them with the section. + + The smallest segment of a track is to generate a Line with the coordinates of + two neighboring detections in the track. + + """ + + def __init__( + self, + calculate_direction_vector_: Callable[ + [float, float, float, float], DirectionVector2D + ] = calculate_direction_vector, + ) -> None: + self._calculate_direction_vector = calculate_direction_vector_ + + def intersect( + self, + track_dataset: TrackDataset, + sections: Iterable[Section], + event_builder: EventBuilder, + ) -> list[Event]: + sections_grouped_by_offset = group_sections_by_offset( + sections, EventType.SECTION_ENTER + ) + events = [] + for offset, section_group in sections_grouped_by_offset.items(): + events.extend( + self.__do_intersect(track_dataset, section_group, offset, event_builder) + ) + return events + + def __do_intersect( + self, + track_dataset: TrackDataset, + sections: list[Section], + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder, + ) -> list[Event]: + intersection_result = track_dataset.intersection_points(sections, offset) + + events: list[Event] = [] + for track_id, intersection_points in intersection_result.items(): + if not (track := track_dataset.get_for(track_id)): + raise IntersectionError( + "Track not found. Unable to create intersection event " + f"for track {track_id}." + ) + event_builder.add_road_user_type(track.classification) + for section_id, intersection_point in intersection_points: + event_builder.add_section_id(section_id) + detection = track.detections[intersection_point.index] + current_coord = detection.get_coordinate(offset) + prev_coord = track.detections[ + intersection_point.index - 1 + ].get_coordinate(offset) + direction_vector = self._calculate_direction_vector( + prev_coord.x, + prev_coord.y, + current_coord.x, + current_coord.y, + ) + event_builder.add_event_type(EventType.SECTION_ENTER) + event_builder.add_direction_vector(direction_vector) + event_builder.add_event_coordinate(current_coord.x, current_coord.y) + events.append(event_builder.create_event(detection)) + return events + + +class ShapelyIntersectAreaByTrackPoints(Intersector): + def __init__( + self, + calculate_direction_vector_: Callable[ + [float, float, float, float], DirectionVector2D + ] = calculate_direction_vector, + ) -> None: + self._calculate_direction_vector = calculate_direction_vector_ + + def intersect( + self, + track_dataset: TrackDataset, + sections: Iterable[Section], + event_builder: EventBuilder, + ) -> list[Event]: + sections_grouped_by_offset = group_sections_by_offset( + sections, EventType.SECTION_ENTER + ) + events = [] + for offset, section_group in sections_grouped_by_offset.items(): + events.extend( + self.__do_intersect(track_dataset, section_group, offset, event_builder) + ) + return events + + def __do_intersect( + self, + track_dataset: TrackDataset, + sections: list[Section], + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder, + ) -> list[Event]: + contained_by_sections_result = track_dataset.contained_by_sections( + sections, offset + ) + + events = [] + for ( + track_id, + contained_by_sections_masks, + ) in contained_by_sections_result.items(): + if not (track := track_dataset.get_for(track_id)): + raise IntersectionError( + "Track not found. Unable to create intersection event " + f"for track {track_id}." + ) + track_detections = track.detections + for section_id, section_entered_mask in contained_by_sections_masks: + event_builder.add_section_id(section_id) + event_builder.add_road_user_type(track.classification) + + track_starts_inside_area = section_entered_mask[0] + if track_starts_inside_area: + first_detection = track_detections[0] + first_coord = first_detection.get_coordinate(offset) + second_coord = track_detections[1].get_coordinate(offset) + + event_builder.add_event_type(EventType.SECTION_ENTER) + event_builder.add_direction_vector( + self._calculate_direction_vector( + first_coord.x, + first_coord.y, + second_coord.x, + second_coord.y, + ) + ) + event_builder.add_event_coordinate( + first_detection.x, first_detection.y + ) + event = event_builder.create_event(first_detection) + events.append(event) + + section_currently_entered = track_starts_inside_area + for current_index, current_detection in enumerate( + track_detections[1:], start=1 + ): + entered = section_entered_mask[current_index] + if section_currently_entered == entered: + continue + + prev_coord = track_detections[current_index - 1].get_coordinate( + offset + ) + current_coord = current_detection.get_coordinate(offset) + + event_builder.add_direction_vector( + self._calculate_direction_vector( + prev_coord.x, + prev_coord.y, + current_coord.x, + current_coord.y, + ) + ) + event_builder.add_event_coordinate(current_coord.x, current_coord.y) + + if entered: + event_builder.add_event_type(EventType.SECTION_ENTER) + else: + event_builder.add_event_type(EventType.SECTION_LEAVE) + + event = event_builder.create_event(current_detection) + events.append(event) + section_currently_entered = entered + + return events + + +class ShapelyCreateIntersectionEvents: + def __init__( + self, + intersect_line_section: Intersector, + intersect_area_section: Intersector, + track_dataset: TrackDataset, + sections: Iterable[Section], + event_builder: SectionEventBuilder, + ): + self._intersect_line_section = intersect_line_section + self._intersect_area_section = intersect_area_section + self._track_dataset = track_dataset + self._sections = sections + self._event_builder = event_builder + + def create(self) -> list[Event]: + events = [] + line_sections, area_sections = separate_sections(self._sections) + events.extend( + self._intersect_line_section.intersect( + self._track_dataset, line_sections, self._event_builder + ) + ) + events.extend( + self._intersect_area_section.intersect( + self._track_dataset, area_sections, self._event_builder + ) + ) + return events + + +class ShapelyRunIntersect(RunIntersect): + def __init__( + self, + intersect_parallelizer: IntersectParallelizationStrategy, + get_tracks: GetAllTracks, + ) -> None: + self._intersect_parallelizer = intersect_parallelizer + self._get_tracks = get_tracks + + def __call__(self, sections: Iterable[Section]) -> list[Event]: + filtered_tracks = self._get_tracks.as_dataset() + + batches = filtered_tracks.split(self._intersect_parallelizer.num_processes) + + tasks = [(batch, sections) for batch in batches] + return self._intersect_parallelizer.execute(_create_events, tasks) + + +def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Event]: + events = [] + event_builder = SectionEventBuilder() + + create_intersection_events = ShapelyCreateIntersectionEvents( + intersect_line_section=ShapelyIntersectBySmallestTrackSegments(), + intersect_area_section=ShapelyIntersectAreaByTrackPoints(), + track_dataset=tracks, + sections=sections, + event_builder=event_builder, + ) + events.extend(create_intersection_events.create()) + return events + + +def separate_sections( + sections: Iterable[Section], +) -> tuple[Iterable[LineSection], Iterable[Area]]: + line_sections = [] + area_sections = [] + for section in sections: + if isinstance(section, LineSection): + line_sections.append(section) + elif isinstance(section, Area): + area_sections.append(section) + else: + raise TypeError( + "Unable to separate section. " + f"Unknown section type for section {section.name} " + f"with type {type(section)}" + ) + + return line_sections, area_sections diff --git a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py index 4dca0217f..7f25aa9f8 100644 --- a/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py +++ b/OTAnalytics/plugin_intersect/shapely/create_intersection_events.py @@ -3,23 +3,10 @@ from shapely import LineString, Polygon -from OTAnalytics.application.analysis.intersect import ( - RunIntersect, - group_sections_by_offset, -) from OTAnalytics.application.geometry import GeometryBuilder -from OTAnalytics.application.use_cases.track_repository import GetAllTracks -from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder -from OTAnalytics.domain.geometry import ( - DirectionVector2D, - RelativeOffsetCoordinate, - apply_offset, - calculate_direction_vector, -) -from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy -from OTAnalytics.domain.section import Area, LineSection, Section -from OTAnalytics.domain.track import Track, TrackDataset, TrackId -from OTAnalytics.domain.types import EventType +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset +from OTAnalytics.domain.section import Area, LineSection +from OTAnalytics.domain.track import Track, TrackId class ShapelyGeometryBuilder(GeometryBuilder[LineString, Polygon]): @@ -81,269 +68,3 @@ def look_up(self, track: Track) -> LineString: self._table[track.id] = new_line return new_line - - -class IntersectionError(Exception): - pass - - -class ShapelyIntersectBySmallestTrackSegments(Intersector): - """ - Implements the intersection strategy by splitting up the track in its smallest - segments and intersecting each of them with the section. - - The smallest segment of a track is to generate a Line with the coordinates of - two neighboring detections in the track. - - """ - - def __init__( - self, - calculate_direction_vector_: Callable[ - [float, float, float, float], DirectionVector2D - ] = calculate_direction_vector, - ) -> None: - self._calculate_direction_vector = calculate_direction_vector_ - - def intersect( - self, - track_dataset: TrackDataset, - sections: Iterable[Section], - event_builder: EventBuilder, - ) -> list[Event]: - sections_grouped_by_offset = group_sections_by_offset( - sections, EventType.SECTION_ENTER - ) - events = [] - for offset, section_group in sections_grouped_by_offset.items(): - events.extend( - self.__do_intersect(track_dataset, section_group, offset, event_builder) - ) - return events - - def __do_intersect( - self, - track_dataset: TrackDataset, - sections: list[Section], - offset: RelativeOffsetCoordinate, - event_builder: EventBuilder, - ) -> list[Event]: - intersection_result = track_dataset.intersection_points(sections, offset) - - events: list[Event] = [] - for track_id, intersection_points in intersection_result.items(): - if not (track := track_dataset.get_for(track_id)): - raise IntersectionError( - "Track not found. Unable to create intersection event " - f"for track {track_id}." - ) - event_builder.add_road_user_type(track.classification) - for section_id, intersection_point in intersection_points: - event_builder.add_section_id(section_id) - detection = track.detections[intersection_point.index] - current_coord = detection.get_coordinate(offset) - prev_coord = track.detections[ - intersection_point.index - 1 - ].get_coordinate(offset) - direction_vector = self._calculate_direction_vector( - prev_coord.x, - prev_coord.y, - current_coord.x, - current_coord.y, - ) - event_builder.add_event_type(EventType.SECTION_ENTER) - event_builder.add_direction_vector(direction_vector) - event_builder.add_event_coordinate(current_coord.x, current_coord.y) - events.append(event_builder.create_event(detection)) - return events - - -class ShapelyIntersectAreaByTrackPoints(Intersector): - def __init__( - self, - calculate_direction_vector_: Callable[ - [float, float, float, float], DirectionVector2D - ] = calculate_direction_vector, - ) -> None: - self._calculate_direction_vector = calculate_direction_vector_ - - def intersect( - self, - track_dataset: TrackDataset, - sections: Iterable[Section], - event_builder: EventBuilder, - ) -> list[Event]: - sections_grouped_by_offset = group_sections_by_offset( - sections, EventType.SECTION_ENTER - ) - events = [] - for offset, section_group in sections_grouped_by_offset.items(): - events.extend( - self.__do_intersect(track_dataset, section_group, offset, event_builder) - ) - return events - - def __do_intersect( - self, - track_dataset: TrackDataset, - sections: list[Section], - offset: RelativeOffsetCoordinate, - event_builder: EventBuilder, - ) -> list[Event]: - contained_by_sections_result = track_dataset.contained_by_sections( - sections, offset - ) - - events = [] - for ( - track_id, - contained_by_sections_masks, - ) in contained_by_sections_result.items(): - if not (track := track_dataset.get_for(track_id)): - raise IntersectionError( - "Track not found. Unable to create intersection event " - f"for track {track_id}." - ) - track_detections = track.detections - for section_id, section_entered_mask in contained_by_sections_masks: - event_builder.add_section_id(section_id) - event_builder.add_road_user_type(track.classification) - - track_starts_inside_area = section_entered_mask[0] - if track_starts_inside_area: - first_detection = track_detections[0] - first_coord = first_detection.get_coordinate(offset) - second_coord = track_detections[1].get_coordinate(offset) - - event_builder.add_event_type(EventType.SECTION_ENTER) - event_builder.add_direction_vector( - self._calculate_direction_vector( - first_coord.x, - first_coord.y, - second_coord.x, - second_coord.y, - ) - ) - event_builder.add_event_coordinate( - first_detection.x, first_detection.y - ) - event = event_builder.create_event(first_detection) - events.append(event) - - section_currently_entered = track_starts_inside_area - for current_index, current_detection in enumerate( - track_detections[1:], start=1 - ): - entered = section_entered_mask[current_index] - if section_currently_entered == entered: - continue - - prev_coord = track_detections[current_index - 1].get_coordinate( - offset - ) - current_coord = current_detection.get_coordinate(offset) - - event_builder.add_direction_vector( - self._calculate_direction_vector( - prev_coord.x, - prev_coord.y, - current_coord.x, - current_coord.y, - ) - ) - event_builder.add_event_coordinate(current_coord.x, current_coord.y) - - if entered: - event_builder.add_event_type(EventType.SECTION_ENTER) - else: - event_builder.add_event_type(EventType.SECTION_LEAVE) - - event = event_builder.create_event(current_detection) - events.append(event) - section_currently_entered = entered - - return events - - -class ShapelyCreateIntersectionEvents: - def __init__( - self, - intersect_line_section: Intersector, - intersect_area_section: Intersector, - track_dataset: TrackDataset, - sections: Iterable[Section], - event_builder: SectionEventBuilder, - ): - self._intersect_line_section = intersect_line_section - self._intersect_area_section = intersect_area_section - self._track_dataset = track_dataset - self._sections = sections - self._event_builder = event_builder - - def create(self) -> list[Event]: - events = [] - line_sections, area_sections = separate_sections(self._sections) - events.extend( - self._intersect_line_section.intersect( - self._track_dataset, line_sections, self._event_builder - ) - ) - events.extend( - self._intersect_area_section.intersect( - self._track_dataset, area_sections, self._event_builder - ) - ) - return events - - -class ShapelyRunIntersect(RunIntersect): - def __init__( - self, - intersect_parallelizer: IntersectParallelizationStrategy, - get_tracks: GetAllTracks, - ) -> None: - self._intersect_parallelizer = intersect_parallelizer - self._get_tracks = get_tracks - - def __call__(self, sections: Iterable[Section]) -> list[Event]: - filtered_tracks = self._get_tracks.as_dataset() - - batches = filtered_tracks.split(self._intersect_parallelizer.num_processes) - - tasks = [(batch, sections) for batch in batches] - return self._intersect_parallelizer.execute(_create_events, tasks) - - -def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Event]: - events = [] - event_builder = SectionEventBuilder() - - create_intersection_events = ShapelyCreateIntersectionEvents( - intersect_line_section=ShapelyIntersectBySmallestTrackSegments(), - intersect_area_section=ShapelyIntersectAreaByTrackPoints(), - track_dataset=tracks, - sections=sections, - event_builder=event_builder, - ) - events.extend(create_intersection_events.create()) - return events - - -def separate_sections( - sections: Iterable[Section], -) -> tuple[Iterable[LineSection], Iterable[Area]]: - line_sections = [] - area_sections = [] - for section in sections: - if isinstance(section, LineSection): - line_sections.append(section) - elif isinstance(section, Area): - area_sections.append(section) - else: - raise TypeError( - "Unable to separate section. " - f"Unknown section type for section {section.name} " - f"with type {type(section)}" - ) - - return line_sections, area_sections diff --git a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py similarity index 99% rename from tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py rename to tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index f6671ab82..4cf019abd 100644 --- a/tests/OTAnalytics/plugin_intersect/shapely/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -3,6 +3,11 @@ import pytest +from OTAnalytics.application.use_cases.create_intersection_events import ( + ShapelyIntersectAreaByTrackPoints, + ShapelyIntersectBySmallestTrackSegments, + separate_sections, +) from OTAnalytics.domain.geometry import ( Coordinate, DirectionVector2D, @@ -18,11 +23,6 @@ ) from OTAnalytics.domain.track import Detection, IntersectionPoint, Track, TrackDataset from OTAnalytics.domain.types import EventType -from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( - ShapelyIntersectAreaByTrackPoints, - ShapelyIntersectBySmallestTrackSegments, - separate_sections, -) from tests.conftest import TrackBuilder diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 5f5c75677..e6cd2fbc8 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -28,6 +28,9 @@ SimpleCreateIntersectionEvents, SimpleCreateSceneEvents, ) +from OTAnalytics.application.use_cases.create_intersection_events import ( + ShapelyRunIntersect, +) from OTAnalytics.application.use_cases.cut_tracks_with_sections import ( CutTracksIntersectingSection, ) @@ -57,9 +60,6 @@ ByMaxConfidence, PythonTrackDataset, ) -from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( - ShapelyRunIntersect, -) from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTrackSegmentBuilder, From bdc0cf07f10fbe1269858bd64dac08d90e819ab3 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:31:46 +0100 Subject: [PATCH 071/107] Rename classes --- OTAnalytics/application/analysis/intersect.py | 1 - .../use_cases/create_intersection_events.py | 14 +++++++------- .../use_cases/test_create_intersection_events.py | 16 ++++++++-------- tests/OTAnalytics/plugin_ui/test_cli.py | 4 ++-- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index e7c457715..bbd1206e4 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -22,7 +22,6 @@ class RunIntersect(ABC): @abstractmethod def __call__(self, sections: Iterable[Section]) -> list[Event]: raise NotImplementedError - # bla class TracksIntersectingSections(ABC): diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 73f4ee175..1b4920815 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -18,7 +18,7 @@ from OTAnalytics.domain.types import EventType -class ShapelyIntersectBySmallestTrackSegments(Intersector): +class IntersectBySmallestTrackSegments(Intersector): """ Implements the intersection strategy by splitting up the track in its smallest segments and intersecting each of them with the section. @@ -89,7 +89,7 @@ def __do_intersect( return events -class ShapelyIntersectAreaByTrackPoints(Intersector): +class IntersectAreaByTrackPoints(Intersector): def __init__( self, calculate_direction_vector_: Callable[ @@ -196,7 +196,7 @@ def __do_intersect( return events -class ShapelyCreateIntersectionEvents: +class RunCreateIntersectionEvents: def __init__( self, intersect_line_section: Intersector, @@ -227,7 +227,7 @@ def create(self) -> list[Event]: return events -class ShapelyRunIntersect(RunIntersect): +class BatchedTracksRunIntersect(RunIntersect): def __init__( self, intersect_parallelizer: IntersectParallelizationStrategy, @@ -249,9 +249,9 @@ def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Ev events = [] event_builder = SectionEventBuilder() - create_intersection_events = ShapelyCreateIntersectionEvents( - intersect_line_section=ShapelyIntersectBySmallestTrackSegments(), - intersect_area_section=ShapelyIntersectAreaByTrackPoints(), + create_intersection_events = RunCreateIntersectionEvents( + intersect_line_section=IntersectBySmallestTrackSegments(), + intersect_area_section=IntersectAreaByTrackPoints(), track_dataset=tracks, sections=sections, event_builder=event_builder, diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 4cf019abd..290efa818 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -4,8 +4,8 @@ import pytest from OTAnalytics.application.use_cases.create_intersection_events import ( - ShapelyIntersectAreaByTrackPoints, - ShapelyIntersectBySmallestTrackSegments, + IntersectAreaByTrackPoints, + IntersectBySmallestTrackSegments, separate_sections, ) from OTAnalytics.domain.geometry import ( @@ -271,9 +271,9 @@ def test_case_line_section_no_intersection(track: Track) -> _TestCase: return _TestCase(track, track_dataset, section, [], []) -class TestShapelyIntersectBySmallestTrackSegments: - def _create_intersector(self) -> ShapelyIntersectBySmallestTrackSegments: - return ShapelyIntersectBySmallestTrackSegments() +class TestIntersectBySmallestTrackSegments: + def _create_intersector(self) -> IntersectBySmallestTrackSegments: + return IntersectBySmallestTrackSegments() @pytest.mark.parametrize( "test_case_name", @@ -303,9 +303,9 @@ def test_intersect( test_case.assert_valid(result_events, event_builder) -class TestShapelyIntersectAreaByTrackPoints: - def _create_intersector(self) -> ShapelyIntersectAreaByTrackPoints: - return ShapelyIntersectAreaByTrackPoints() +class TestIntersectAreaByTrackPoints: + def _create_intersector(self) -> IntersectAreaByTrackPoints: + return IntersectAreaByTrackPoints() @pytest.fixture def test_case_track_starts_outside_section( diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index e6cd2fbc8..48000ac55 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -29,7 +29,7 @@ SimpleCreateSceneEvents, ) from OTAnalytics.application.use_cases.create_intersection_events import ( - ShapelyRunIntersect, + BatchedTracksRunIntersect, ) from OTAnalytics.application.use_cases.cut_tracks_with_sections import ( CutTracksIntersectingSection, @@ -277,7 +277,7 @@ def cli_dependencies(self) -> dict[str, Any]: clear_all_events = ClearAllEvents(event_repository) create_intersection_events = SimpleCreateIntersectionEvents( - ShapelyRunIntersect( + BatchedTracksRunIntersect( MultiprocessingIntersectParallelization(), get_all_tracks, ), From 09af1a346a92f1a84972836a62f82387e4fd4686 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:43:47 +0100 Subject: [PATCH 072/107] Let DataFrame filter predicate directly return filtered DataFrame instead of boolean mask --- OTAnalytics/plugin_filter/dataframe_filter.py | 62 ++++++++++--------- .../plugin_filter/test_dataframe_filter.py | 32 +++++----- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/OTAnalytics/plugin_filter/dataframe_filter.py b/OTAnalytics/plugin_filter/dataframe_filter.py index 682c2f494..5b1c8937b 100644 --- a/OTAnalytics/plugin_filter/dataframe_filter.py +++ b/OTAnalytics/plugin_filter/dataframe_filter.py @@ -6,56 +6,59 @@ from OTAnalytics.domain.filter import Conjunction, Filter, FilterBuilder, Predicate -class DataFrameConjunction(Conjunction[DataFrame, Series]): +class DataFrameConjunction(Conjunction[DataFrame, DataFrame]): """Represents the conjunction of two DataFrame predicates. Args: - first_predicate (Predicate[DataFrame, Series]): first predicate to + first_predicate (Predicate[DataFrame, DataFrame]): first predicate to conjunct with - second_predicate (Predicate[DataFrame, Series]): second predicate to + second_predicate (Predicate[DataFrame, DataFrame]): second predicate to conjunct with """ def __init__( self, - first_predicate: Predicate[DataFrame, Series], - second_predicate: Predicate[DataFrame, Series], + first_predicate: Predicate[DataFrame, DataFrame], + second_predicate: Predicate[DataFrame, DataFrame], ) -> None: super().__init__(first_predicate, second_predicate) - def test(self, to_test: DataFrame) -> Series: - return (self._first_predicate.test(to_test)) & ( - self._second_predicate.test(to_test) - ) + def test(self, to_test: DataFrame) -> DataFrame: + return self._second_predicate.test(self._first_predicate.test(to_test)) def conjunct_with( - self, other: Predicate[DataFrame, Series] - ) -> Predicate[DataFrame, Series]: + self, other: Predicate[DataFrame, DataFrame] + ) -> Predicate[DataFrame, DataFrame]: return DataFrameConjunction(self, other) -class DataFramePredicate(Predicate[DataFrame, Series]): +class DataFramePredicate(Predicate[DataFrame, DataFrame]): + """Checks DataFrame entries against predicate. + + Entries that do not fulfill predicate are filtered out. + """ + def conjunct_with( - self, other: Predicate[DataFrame, Series] - ) -> Predicate[DataFrame, Series]: + self, other: Predicate[DataFrame, DataFrame] + ) -> Predicate[DataFrame, DataFrame]: return DataFrameConjunction(self, other) -class DataFrameFilter(Filter[DataFrame, Series]): - def __init__(self, predicate: Predicate[DataFrame, Series]) -> None: +class DataFrameFilter(Filter[DataFrame, DataFrame]): + def __init__(self, predicate: Predicate[DataFrame, DataFrame]) -> None: """A `DataFrame` filter. Args: - predicate (Predicate[DataFrame, Series]): the predicate to test + predicate (Predicate[DataFrame, DataFrame]): the predicate to test the DataFrame against """ self._predicate = predicate def apply(self, data: Iterable[DataFrame]) -> Iterable[DataFrame]: - return [datum[self._predicate.test(datum)] for datum in data] + return [self._predicate.test(df) for df in data] -class NoOpDataFrameFilter(Filter[DataFrame, Series]): +class NoOpDataFrameFilter(Filter[DataFrame, DataFrame]): """Returns the DataFrame as is without any filtering.""" def apply(self, iterable: Iterable[DataFrame]) -> Iterable[DataFrame]: @@ -81,12 +84,13 @@ def __init__( self.column_name: str = column_name self._start_date = start_date - def test(self, to_test: DataFrame) -> Series: + def test(self, to_test: DataFrame) -> DataFrame: # TODO: Only works for DataFrames that have track id and occurrence as # multi-index - return ( + + return to_test[ to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) >= self._start_date - ) + ] class DataFrameEndsBeforeOrAtDate(DataFramePredicate): @@ -105,10 +109,12 @@ def __init__( self.column_name: str = column_name self._end_date = end_date - def test(self, to_test: DataFrame) -> Series: + def test(self, to_test: DataFrame) -> DataFrame: # TODO: Only works for DataFrames that have track id and occurrence as # multi-index - return to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) <= self._end_date + return to_test[ + to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) <= self._end_date + ] class DataFrameHasClassifications(DataFramePredicate): @@ -127,16 +133,16 @@ def __init__( self._column_name = column_name self._classifications = classifications - def test(self, to_test: DataFrame) -> Series: - return to_test[self._column_name].isin(self._classifications) + def test(self, to_test: DataFrame) -> DataFrame: + return to_test[to_test[self._column_name].isin(self._classifications)] -class DataFrameFilterBuilder(FilterBuilder[DataFrame, Series]): +class DataFrameFilterBuilder(FilterBuilder[DataFrame, DataFrame]): """A builder used to build a `DataFrameFilter`.""" def __init__(self) -> None: super().__init__() - self._complex_predicate: Optional[Predicate[DataFrame, Series]] = None + self._complex_predicate: Optional[Predicate[DataFrame, DataFrame]] = None self._classification_column: Optional[str] = None self._occurrence_column: Optional[str] = None diff --git a/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py b/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py index 933b1f786..20cfc284a 100644 --- a/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py +++ b/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py @@ -3,8 +3,9 @@ from unittest.mock import Mock import pytest -from pandas import DataFrame, Series +from pandas import DataFrame +from OTAnalytics.domain import track from OTAnalytics.domain.track import ( CLASSIFICATION, FRAME, @@ -39,12 +40,12 @@ def convert_tracks_to_dataframe(tracks: Iterable[Track]) -> DataFrame: for current_track in tracks: detections.extend(current_track.detections) prepared = [detection.to_dict() for detection in detections] - converted = DataFrame(prepared) + converted = DataFrame(prepared).set_index([track.TRACK_ID, track.OCCURRENCE]) return converted.sort_values([TRACK_ID, FRAME]) @pytest.fixture -def track(track_builder: TrackBuilder) -> Track: +def simple_track(track_builder: TrackBuilder) -> Track: track_builder.add_occurrence(2000, 1, 2, 0, 0, 0, 0) track_builder.append_detection() track_builder.append_detection() @@ -55,33 +56,33 @@ def track(track_builder: TrackBuilder) -> Track: @pytest.fixture -def track_dataframe(track: Track) -> DataFrame: - return convert_tracks_to_dataframe([track]) +def track_dataframe(simple_track: Track) -> DataFrame: + return convert_tracks_to_dataframe([simple_track]) class TestDataFramePredicates: @pytest.mark.parametrize( - "predicate, expected_result", + "predicate, expected_mask", [ ( DataFrameStartsAtOrAfterDate( OCCURRENCE, datetime(2000, 1, 1, tzinfo=timezone.utc) ), - Series([True, True, True, True, True]), + [True, True, True, True, True], ), ( DataFrameStartsAtOrAfterDate( OCCURRENCE, datetime(2000, 1, 10, tzinfo=timezone.utc) ), - Series([False, False, False, False, False]), + [False, False, False, False, False], ), ( DataFrameHasClassifications(CLASSIFICATION, {"car", "truck"}), - Series([True, True, True, True, True]), + [True, True, True, True, True], ), ( DataFrameHasClassifications(CLASSIFICATION, {"bicycle", "truck"}), - Series([False, False, False, False, False]), + [False, False, False, False, False], ), ( DataFrameStartsAtOrAfterDate( @@ -89,7 +90,7 @@ class TestDataFramePredicates: ).conjunct_with( DataFrameHasClassifications(CLASSIFICATION, {"car", "truck"}), ), - Series([True, True, True, True, True]), + [True, True, True, True, True], ), ( DataFrameStartsAtOrAfterDate( @@ -97,17 +98,20 @@ class TestDataFramePredicates: ).conjunct_with( DataFrameHasClassifications(CLASSIFICATION, {"car", "truck"}), ), - Series([False, False, False, False, False]), + [False, False, False, False, False], ), ], ) def test_predicate( self, predicate: DataFramePredicate, - expected_result: Series, + expected_mask: list[bool], track_dataframe: DataFrame, ) -> None: - assert predicate.test(track_dataframe).equals(expected_result) + result = predicate.test(track_dataframe) + expected_result = track_dataframe[expected_mask] + + assert result.equals(expected_result) class TestDataFrameFilter: From 371c11decb15526caed73fa3a1f4d6c22ac0dbdc Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:44:53 +0100 Subject: [PATCH 073/107] Fix unit test --- .../track_visualization/test_track_viz.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 442c77198..0d3987f3d 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -6,6 +6,7 @@ from OTAnalytics.application.datastore import Datastore from OTAnalytics.application.state import ObservableProperty, TrackViewState +from OTAnalytics.domain import track from OTAnalytics.domain.event import Event from OTAnalytics.domain.filter import Filter, FilterBuilder, FilterElement from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository @@ -234,8 +235,8 @@ def check_expected_ids( assert expected_detections == len(provider._cache_df) assert len(expected_tracks) == len(cached_ids) - for track in expected_tracks: - assert track.id.id in cached_ids + for _track in expected_tracks: + assert _track.id.id in cached_ids def test_notify_tracks_clear_cache(self, track_1: Track) -> None: """Test clearing cache.""" @@ -448,8 +449,15 @@ def test_plot( class TestDataFrameProviderFilter: @pytest.fixture def filter_input(self) -> DataFrame: - d = {TRACK_ID: [1, 2]} - return DataFrame(data=d) + first_occurrence = datetime(2000, 1, 1, 1) + second_occurrence = datetime(2000, 1, 1, 2) + d = { + track.TRACK_ID: ["1", "2"], + track.OCCURRENCE: [first_occurrence, second_occurrence], + "data": [Mock(), Mock()], + } + df = DataFrame(data=d) + return df.set_index([track.TRACK_ID, track.OCCURRENCE]) @pytest.fixture def filter_result(self) -> Mock: @@ -485,17 +493,18 @@ def track_view_state(self, observable_filter_element: Mock) -> Mock: track_view_state.filter_element = observable_filter_element return track_view_state - def test_filter_by_id(self, data_provider: Mock) -> None: + def test_filter_by_id(self, data_provider: Mock, filter_input: DataFrame) -> None: id_filter = Mock(spec=TrackIdProvider) track_id = Mock(spec=TrackId) - track_id.id = 1 + track_id.id = "1" id_filter.get_ids.return_value = [track_id] filter_by_id = FilterById(data_provider, id_filter) result = filter_by_id.get_data() + expected = filter_input.drop("2") - assert result.equals(DataFrame(data={TRACK_ID: [1]})) + assert result.equals(expected) data_provider.get_data.assert_called_once() id_filter.get_ids.assert_called_once() From 5147b030ac055fe255389f62154a58ca64c063ef Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:49:35 +0100 Subject: [PATCH 074/107] Use PandasTrackDataset as underlying implementation of track repository --- OTAnalytics/plugin_ui/main_application.py | 35 ++++++++++--------- .../plugin_ui/visualization/visualization.py | 15 ++++---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 6312bf344..e4898dc52 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -45,6 +45,9 @@ SimpleCreateIntersectionEvents, SimpleCreateSceneEvents, ) +from OTAnalytics.application.use_cases.create_intersection_events import ( + BatchedTracksRunIntersect, +) from OTAnalytics.application.use_cases.cut_tracks_with_sections import ( CutTracksIntersectingSection, ) @@ -98,12 +101,10 @@ from OTAnalytics.domain.section import SectionRepository from OTAnalytics.domain.track import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository -from OTAnalytics.plugin_datastore.python_track_store import ( - ByMaxConfidence, - PythonTrackDataset, -) -from OTAnalytics.plugin_intersect.shapely.create_intersection_events import ( - ShapelyRunIntersect, +from OTAnalytics.plugin_datastore.python_track_store import ByMaxConfidence +from OTAnalytics.plugin_datastore.track_store import ( + PandasByMaxConfidence, + PandasTrackDataset, ) from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( @@ -130,9 +131,9 @@ OtFlowParser, OttrkParser, OttrkVideoParser, - PythonDetectionParser, SimpleVideoParser, ) +from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser from OTAnalytics.plugin_progress.tqdm_progressbar import TqdmBuilder from OTAnalytics.plugin_prototypes.eventlist_exporter.eventlist_exporter import ( AVAILABLE_EVENTLIST_EXPORTERS, @@ -550,18 +551,18 @@ def _create_datastore( ) def _create_track_repository(self) -> TrackRepository: - # return TrackRepository(PandasTrackDataset.from_list([])) - return TrackRepository(PythonTrackDataset()) + return TrackRepository(PandasTrackDataset.from_list([])) + # return TrackRepository(PythonTrackDataset()) def _create_track_parser(self, track_repository: TrackRepository) -> TrackParser: - # calculator = PandasByMaxConfidence() - # detection_parser = PandasDetectionParser( - # calculator, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT - # ) - calculator = ByMaxConfidence() - detection_parser = PythonDetectionParser( - calculator, track_repository, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT + calculator = PandasByMaxConfidence() + detection_parser = PandasDetectionParser( + calculator, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT ) + # calculator = ByMaxConfidence() + # detection_parser = PythonDetectionParser( + # noqa calculator, track_repository, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT + # ) return OttrkParser(detection_parser) def _create_section_repository(self) -> SectionRepository: @@ -651,7 +652,7 @@ def _create_use_case_create_intersection_events( @staticmethod def _create_intersect(get_tracks: GetAllTracks, num_processes: int) -> RunIntersect: - return ShapelyRunIntersect( + return BatchedTracksRunIntersect( intersect_parallelizer=MultiprocessingIntersectParallelization( num_processes ), diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 224c3e12d..b8575b310 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -29,7 +29,6 @@ SimpleTracksIntersectingSections, ) from OTAnalytics.plugin_prototypes.track_visualization.track_viz import ( - CachedPandasTrackProvider, ColorPaletteProvider, EventToFlowResolver, FilterByClassification, @@ -321,18 +320,18 @@ def _create_pandas_track_provider( self, progressbar: ProgressbarBuilder ) -> PandasTrackProvider: dataframe_filter_builder = self._create_dataframe_filter_builder() - # return PandasTrackProvider( - # self._track_repository, - # self._track_view_state, - # dataframe_filter_builder, - # progressbar, - # ) - return CachedPandasTrackProvider( + return PandasTrackProvider( self._track_repository, self._track_view_state, dataframe_filter_builder, progressbar, ) + # return CachedPandasTrackProvider( + # self._track_repository, + # self._track_view_state, + # dataframe_filter_builder, + # progressbar, + # ) def _wrap_pandas_track_offset_provider( self, other: PandasDataFrameProvider From 2718a0289a28d6b322f834bbd4e1c59acef761f0 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:50:11 +0100 Subject: [PATCH 075/107] Fix profile test --- .../track_geometry_store/test_pygeos_store.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index cfae74add..73481441c 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -654,22 +654,22 @@ def tracks_15min(self, test_data_dir: Path) -> TrackDataset: return parse_result.tracks @pytest.fixture - def sections(self, test_data_dir: Path) -> Iterable[Section]: + def sections(self, test_data_dir: Path) -> list[Section]: flow_file = test_data_dir / "OTCamera19_FR20_2023-05-24.otflow" flow_parser = OtFlowParser() sections, flows = flow_parser.parse(flow_file) - return sections + return list(sections) @pytest.mark.skip def test_profile( self, benchmark: BenchmarkFixture, tracks_15min: TrackDataset, - sections: Iterable[Section], + sections: list[Section], ) -> None: benchmark.pedantic( tracks_15min.intersecting_tracks, - args=(sections,), + args=(sections, sections[0].get_offset(EventType.SECTION_ENTER)), rounds=self.ROUNDS, iterations=self.ITERATIONS, warmup_rounds=self.WARMUP_ROUNDS, From 779302e5767c62b9116b42ca693635af9fa00407 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 7 Dec 2023 16:21:24 +0100 Subject: [PATCH 076/107] Switch to pandas track store implementation in benchmark --- tests/benchmark_otanalytics.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index 8f094b626..ca7f92b36 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -254,11 +254,11 @@ class TestBenchmarkTrackParser: def test_load_15min( self, benchmark: BenchmarkFixture, - python_track_parser: TrackParser, + pandas_track_parser: TrackParser, track_file_15min: Path, ) -> None: benchmark.pedantic( - python_track_parser.parse, + pandas_track_parser.parse, args=(track_file_15min,), rounds=self.ROUNDS, iterations=self.ITERATIONS, @@ -268,7 +268,7 @@ def test_load_15min( def test_load_2hours( self, benchmark: BenchmarkFixture, - python_track_parser: TrackParser, + pandas_track_parser: TrackParser, track_files_2hours: list[Path], ) -> None: def _parse_2hours(parser: TrackParser, ottrk_files: list[Path]) -> None: @@ -278,7 +278,7 @@ def _parse_2hours(parser: TrackParser, ottrk_files: list[Path]) -> None: benchmark.pedantic( _parse_2hours, args=( - python_track_parser, + pandas_track_parser, track_files_2hours, ), rounds=self.ROUNDS, @@ -332,10 +332,10 @@ class TestBenchmarkTracksIntersectingSections: def test_15min( self, benchmark: BenchmarkFixture, - python_track_repo_15min: tuple[TrackRepository, DetectionMetadata], + pandas_track_repo_15min: tuple[TrackRepository, DetectionMetadata], section_flow_repo_setup: tuple[SectionRepository, FlowRepository], ) -> None: - track_repository, _ = python_track_repo_15min + track_repository, _ = pandas_track_repo_15min section_repository, flow_repository = section_flow_repo_setup use_case = _build_tracks_intersecting_sections(track_repository) @@ -350,10 +350,10 @@ def test_15min( def test_2hours( self, benchmark: BenchmarkFixture, - python_track_repo_2hours: tuple[TrackRepository, DetectionMetadata], + pandas_track_repo_2hours: tuple[TrackRepository, DetectionMetadata], section_flow_repo_setup: tuple[SectionRepository, FlowRepository], ) -> None: - track_repository, _ = python_track_repo_2hours + track_repository, _ = pandas_track_repo_2hours section_repository, flow_repository = section_flow_repo_setup use_case = _build_tracks_intersecting_sections(track_repository) @@ -414,12 +414,12 @@ class TestBenchmarkCreateEvents: def test_15min( self, benchmark: BenchmarkFixture, - python_track_repo_15min: tuple[TrackRepository, DetectionMetadata], + pandas_track_repo_15min: tuple[TrackRepository, DetectionMetadata], section_flow_repo_setup: tuple[SectionRepository, FlowRepository], event_repository: EventRepository, clear_events: ClearAllEvents, ) -> None: - track_repository, _ = python_track_repo_15min + track_repository, _ = pandas_track_repo_15min section_repository, flow_repository = section_flow_repo_setup create_events = _build_create_events( track_repository, section_repository, event_repository @@ -435,12 +435,12 @@ def test_15min( def test_2hours( self, benchmark: BenchmarkFixture, - python_track_repo_2hours: tuple[TrackRepository, DetectionMetadata], + pandas_track_repo_2hours: tuple[TrackRepository, DetectionMetadata], section_flow_repo_setup: tuple[SectionRepository, FlowRepository], event_repository: EventRepository, clear_events: ClearAllEvents, ) -> None: - track_repository, _ = python_track_repo_2hours + track_repository, _ = pandas_track_repo_2hours section_repository, flow_repository = section_flow_repo_setup create_events = _build_create_events( track_repository, section_repository, event_repository From 0313b393019a9e62e882dc7a794e15a8a45875a9 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:21:03 +0100 Subject: [PATCH 077/107] Use vectorized implementation of line_locate_point with dataframes --- OTAnalytics/domain/track.py | 5 + .../track_geometry_store/pygeos_store.py | 69 +++++++++- OTAnalytics/plugin_datastore/track_store.py | 48 ++++--- OTAnalytics/plugin_parser/pandas_parser.py | 6 +- OTAnalytics/plugin_ui/main_application.py | 13 +- .../OTAnalytics/plugin_datastore/conftest.py | 10 +- .../plugin_datastore/test_track_store.py | 118 ++++++++++++------ .../track_geometry_store/test_pygeos_store.py | 43 ++++++- .../plugin_parser/test_pandas_parser.py | 21 +++- tests/benchmark_otanalytics.py | 47 ++++--- tests/conftest.py | 7 +- 11 files changed, 299 insertions(+), 88 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index a07793256..cd9d2e9cd 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -707,6 +707,11 @@ def track_ids(self) -> set[str]: def offset(self) -> RelativeOffsetCoordinate: raise NotImplementedError + @property + @abstractmethod + def empty(self) -> bool: + raise NotImplementedError + @staticmethod @abstractmethod def from_track_dataset( diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 9af791d86..3c8f75265 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -3,7 +3,7 @@ from itertools import chain from typing import Any, Iterable, Literal, TypedDict -from pandas import DataFrame +from pandas import DataFrame, Series, concat from pygeos import ( Geometry, contains, @@ -19,6 +19,7 @@ prepare, ) +from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( @@ -28,6 +29,7 @@ TrackGeometryDataset, TrackId, ) +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset TRACK_ID = "track_id" GEOMETRY = "geom" @@ -133,6 +135,13 @@ def from_track_dataset( ) -> TrackGeometryDataset: if len(dataset) == 0: return PygeosTrackGeometryDataset(offset) + if isinstance(dataset, PandasTrackDataset): + return PygeosTrackGeometryDataset( + offset, + PygeosTrackGeometryDataset.__create_entries_from_dataframe( + dataset, offset + ), + ) track_geom_df = DataFrame.from_dict( PygeosTrackGeometryDataset._create_entries(dataset, offset), columns=COLUMNS, @@ -158,12 +167,12 @@ def _create_entries( dict: the entries. """ entries = dict() - for track in tracks: - if len(track.detections) < 2: + for _track in tracks: + if len(_track.detections) < 2: # Disregard single detection tracks continue - track_id = track.id.id - geometry = create_pygeos_track(track, offset) + track_id = _track.id.id + geometry = create_pygeos_track(_track, offset) projection = [ line_locate_point(geometry, points(p)) for p in get_coordinates(geometry) @@ -174,14 +183,49 @@ def _create_entries( } return entries + @staticmethod + def __create_entries_from_dataframe( + track_dataset: PandasTrackDataset, + offset: RelativeOffsetCoordinate, + ) -> DataFrame: + track_size_mask = track_dataset._dataset.groupby(level=0).transform("size") + filtered_tracks = track_dataset._dataset[track_size_mask > 1] + + if offset == BASE_GEOMETRY: + new_x = filtered_tracks[track.X] + new_y = filtered_tracks[track.Y] + else: + new_x = filtered_tracks[track.X] + offset.x * filtered_tracks[track.W] + new_y = filtered_tracks[track.Y] + offset.y * filtered_tracks[track.H] + tracks = concat([new_x, new_y], keys=[track.X, track.Y], axis=1).groupby( + level=0, group_keys=True + ) + geometries = tracks.agg(list).apply( + lambda coords: linestrings(tuple(zip(coords[track.X], coords[track.Y]))), + axis=1, + ) + projections = tracks.apply(calculate_projection) + result = concat([geometries, projections], keys=COLUMNS, axis=1) + return result + def add_all(self, tracks: Iterable[Track]) -> TrackGeometryDataset: if self.empty: + if isinstance(tracks, PandasTrackDataset): + return PygeosTrackGeometryDataset( + self._offset, + self.__create_entries_from_dataframe(tracks, self.offset), + ) new_entries = self._create_entries(tracks, self._offset) return PygeosTrackGeometryDataset( self._offset, DataFrame.from_dict(new_entries, orient=ORIENTATION_INDEX) ) existing_entries = self.as_dict() - new_entries = self._create_entries(tracks, self._offset) + if isinstance(tracks, PandasTrackDataset): + new_entries = self.__create_entries_from_dataframe( + tracks, self._offset + ).to_dict(orient=ORIENTATION_INDEX) + else: + new_entries = self._create_entries(tracks, self._offset) for track_id, entry in new_entries.items(): existing_entries[track_id] = entry new_dataset = DataFrame.from_dict(existing_entries, orient=ORIENTATION_INDEX) @@ -294,3 +338,16 @@ def contained_by_sections( def as_dict(self) -> dict: return self._dataset[COLUMNS].to_dict(orient=ORIENTATION_INDEX) + + +def calculate_projection(track_df: DataFrame) -> Series: + _track = track_df.reset_index() + x_1 = _track.iloc[:-1][track.X].reset_index(drop=True) + y_1 = _track.iloc[:-1][track.X].reset_index(drop=True) + x_2 = _track.iloc[1:][track.X].reset_index(drop=True) + y_2 = _track.iloc[1:][track.Y].reset_index(drop=True) + + d = ((x_2 - x_1).pow(2) + (y_2 - y_1).pow(2)).pow(1 / 2) + + projection = concat([Series(0), d], ignore_index=True).cumsum() + return projection.agg(list) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 82b598698..ecd55e00b 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -20,9 +20,6 @@ TrackGeometryDataset, TrackId, ) -from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( - PygeosTrackGeometryDataset, -) class PandasDetection(Detection): @@ -154,15 +151,17 @@ def calculate(self, detections: DataFrame) -> DataFrame: class PandasTrackDataset(TrackDataset): def __init__( self, - dataset: DataFrame = DataFrame(), + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + dataset: DataFrame | None = None, geometry_datasets: dict[RelativeOffsetCoordinate, TrackGeometryDataset] | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, - track_geometry_factory: TRACK_GEOMETRY_FACTORY = ( - PygeosTrackGeometryDataset.from_track_dataset - ), ): - self._dataset = dataset + if dataset is None: + self._dataset = DataFrame() + else: + self._dataset = dataset + self._calculator = calculator self._track_geometry_factory = track_geometry_factory if geometry_datasets is None: @@ -175,23 +174,31 @@ def __init__( @staticmethod def from_list( tracks: list[Track], + track_geometry_factory: TRACK_GEOMETRY_FACTORY, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ) -> TrackDataset: return PandasTrackDataset.from_dataframe( - _convert_tracks(tracks), calculator=calculator + _convert_tracks(tracks), + track_geometry_factory, + calculator=calculator, ) @staticmethod def from_dataframe( tracks: DataFrame, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, geometry_dataset: dict[RelativeOffsetCoordinate, TrackGeometryDataset] | None = None, calculator: PandasTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, ) -> TrackDataset: if tracks.empty: - return PandasTrackDataset() + return PandasTrackDataset(track_geometry_factory) classified_tracks = _assign_track_classification(tracks, calculator) - return PandasTrackDataset(classified_tracks, geometry_datasets=geometry_dataset) + return PandasTrackDataset( + track_geometry_factory, + classified_tracks, + geometry_datasets=geometry_dataset, + ) def add_all(self, other: Iterable[Track]) -> TrackDataset: new_tracks = self.__get_tracks(other) @@ -199,16 +206,16 @@ def add_all(self, other: Iterable[Track]) -> TrackDataset: return self if self._dataset.empty: return PandasTrackDataset.from_dataframe( - new_tracks, calculator=self._calculator + new_tracks, self._track_geometry_factory, calculator=self._calculator ) updated_dataset = pandas.concat([self._dataset, new_tracks]).sort_index() new_track_ids = new_tracks.index.levels[LEVEL_TRACK_ID] new_dataset = updated_dataset.loc[new_track_ids] updated_geometry_dataset = self._add_to_geometry_dataset( - PandasTrackDataset.from_dataframe(new_dataset) + PandasTrackDataset.from_dataframe(new_dataset, self._track_geometry_factory) ) return PandasTrackDataset.from_dataframe( - updated_dataset, updated_geometry_dataset + updated_dataset, self._track_geometry_factory, updated_geometry_dataset ) def _add_to_geometry_dataset( @@ -233,13 +240,13 @@ def get_for(self, id: TrackId) -> Optional[Track]: return self.__create_track_flyweight(id.id) def clear(self) -> "TrackDataset": - return PandasTrackDataset() + return PandasTrackDataset(self._track_geometry_factory) def remove(self, track_id: TrackId) -> "TrackDataset": remaining_tracks = self._dataset.drop(track_id.id, errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset({track_id}) return PandasTrackDataset.from_dataframe( - remaining_tracks, updated_geometry_datasets + remaining_tracks, self._track_geometry_factory, updated_geometry_datasets ) def _remove_from_geometry_dataset( @@ -273,7 +280,10 @@ def split(self, batches: int) -> Sequence["TrackDataset"]: batch_geometries = self._get_geometries_for(batch_ids) new_batches.append( PandasTrackDataset.from_dataframe( - batch_dataset, batch_geometries, calculator=self._calculator + batch_dataset, + self._track_geometry_factory, + batch_geometries, + calculator=self._calculator, ) ) return new_batches @@ -300,7 +310,9 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": detection_counts_per_track > length ].index filtered_dataset = self._dataset.loc[filtered_ids] - return PandasTrackDataset(filtered_dataset, calculator=self._calculator) + return PandasTrackDataset( + self._track_geometry_factory, filtered_dataset, calculator=self._calculator + ) def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate diff --git a/OTAnalytics/plugin_parser/pandas_parser.py b/OTAnalytics/plugin_parser/pandas_parser.py index 53630d7e2..c8228aa08 100644 --- a/OTAnalytics/plugin_parser/pandas_parser.py +++ b/OTAnalytics/plugin_parser/pandas_parser.py @@ -5,7 +5,7 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain import track -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasTrackClassificationCalculator, PandasTrackDataset, @@ -22,9 +22,11 @@ class PandasDetectionParser(DetectionParser): def __init__( self, calculator: PandasTrackClassificationCalculator, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, track_length_limit: TrackLengthLimit = DEFAULT_TRACK_LENGTH_LIMIT, ) -> None: self._calculator = calculator + self._track_geometry_factory = track_geometry_factory self._track_length_limit = track_length_limit def parse_tracks( @@ -85,5 +87,5 @@ def _parse_as_dataframe( tracks_to_remain.index.names = [track.TRACK_ID, track.OCCURRENCE] tracks_to_remain = tracks_to_remain.sort_index() return PandasTrackDataset.from_dataframe( - tracks_to_remain, calculator=self._calculator + tracks_to_remain, self._track_geometry_factory, calculator=self._calculator ) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 597c8e4e0..743776768 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -105,6 +105,9 @@ from OTAnalytics.domain.track import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository from OTAnalytics.plugin_datastore.python_track_store import ByMaxConfidence +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import ( PandasByMaxConfidence, PandasTrackDataset, @@ -561,13 +564,19 @@ def _create_datastore( ) def _create_track_repository(self) -> TrackRepository: - return TrackRepository(PandasTrackDataset.from_list([])) + return TrackRepository( + PandasTrackDataset.from_list( + [], PygeosTrackGeometryDataset.from_track_dataset + ) + ) # return TrackRepository(PythonTrackDataset()) def _create_track_parser(self, track_repository: TrackRepository) -> TrackParser: calculator = PandasByMaxConfidence() detection_parser = PandasDetectionParser( - calculator, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT + calculator, + PygeosTrackGeometryDataset.from_track_dataset, + track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, ) # calculator = ByMaxConfidence() # detection_parser = PythonDetectionParser( diff --git a/tests/OTAnalytics/plugin_datastore/conftest.py b/tests/OTAnalytics/plugin_datastore/conftest.py index 58092f812..eb8f91f41 100644 --- a/tests/OTAnalytics/plugin_datastore/conftest.py +++ b/tests/OTAnalytics/plugin_datastore/conftest.py @@ -3,7 +3,10 @@ import pytest -from OTAnalytics.domain.track import Track, TrackGeometryDataset +from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, Track, TrackGeometryDataset +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from tests.conftest import TrackBuilder, assert_equal_track_properties @@ -28,6 +31,11 @@ def assert_track_geometry_dataset_add_all_called_correctly( assert_equal_track_properties(actual_track, expected_track) +@pytest.fixture +def track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: + return PygeosTrackGeometryDataset.from_track_dataset + + @pytest.fixture def first_track() -> Track: track_builder = TrackBuilder() diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index c380df9e5..e41be3616 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -6,11 +6,20 @@ from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate -from OTAnalytics.domain.track import Track, TrackDataset, TrackGeometryDataset, TrackId +from OTAnalytics.domain.track import ( + TRACK_GEOMETRY_FACTORY, + Track, + TrackDataset, + TrackGeometryDataset, + TrackId, +) from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, PythonTrackDataset, ) +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import ( PandasDetection, PandasTrack, @@ -67,10 +76,14 @@ def _create_dataset(self, size: int) -> TrackDataset: for i in range(1, size + 1): tracks.append(self.__build_track(str(i))) - dataset = PandasTrackDataset.from_list(tracks) + dataset = PandasTrackDataset.from_list( + tracks, PygeosTrackGeometryDataset.from_track_dataset + ) return dataset - def test_use_track_classificator(self) -> None: + def test_use_track_classificator( + self, track_geometry_factory: TRACK_GEOMETRY_FACTORY + ) -> None: first_detection_class = "car" track_class = "pedestrian" builder = TrackBuilder() @@ -85,14 +98,14 @@ def test_use_track_classificator(self) -> None: builder.append_detection() track = builder.build_track() - dataset = PandasTrackDataset.from_list([track]) + dataset = PandasTrackDataset.from_list([track], track_geometry_factory) added_track = dataset.get_for(track.id) assert added_track is not None assert added_track.classification == track_class - def test_add(self) -> None: + def test_add(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: builder = TrackBuilder() builder.append_detection() builder.append_detection() @@ -100,8 +113,8 @@ def test_add(self) -> None: builder.append_detection() builder.append_detection() track = builder.build_track() - expected_dataset = PandasTrackDataset.from_list([track]) - dataset = PandasTrackDataset() + expected_dataset = PandasTrackDataset.from_list([track], track_geometry_factory) + dataset = PandasTrackDataset(track_geometry_factory) merged = dataset.add_all(PythonTrackDataset({track.id: track})) @@ -109,21 +122,21 @@ def test_add(self) -> None: for actual, expected in zip(merged, expected_dataset): assert_equal_track_properties(actual, expected) - def test_add_nothing(self) -> None: - dataset = PandasTrackDataset() + def test_add_nothing(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: + dataset = PandasTrackDataset(track_geometry_factory) merged = dataset.add_all(PythonTrackDataset()) assert 0 == len(merged.as_list()) - def test_add_all(self) -> None: + def test_add_all(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") third_track = self.__build_track("3") expected_dataset = PandasTrackDataset.from_list( - [first_track, second_track, third_track] + [first_track, second_track, third_track], track_geometry_factory ) - dataset = PandasTrackDataset.from_list([]) + dataset = PandasTrackDataset.from_list([], track_geometry_factory) assert len(dataset) == 0 dataset = dataset.add_all([first_track]) assert len(dataset) == 1 @@ -139,12 +152,16 @@ def test_add_all(self) -> None: assert_equal_track_properties(actual, expected) assert merged._geometry_datasets == {} - def test_add_two_existing_pandas_datasets(self) -> None: + def test_add_two_existing_pandas_datasets( + self, track_geometry_factory: TRACK_GEOMETRY_FACTORY + ) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - expected_dataset = PandasTrackDataset.from_list([first_track, second_track]) - first = PandasTrackDataset.from_list([first_track]) - second = PandasTrackDataset.from_list([second_track]) + expected_dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) + first = PandasTrackDataset.from_list([first_track], track_geometry_factory) + second = PandasTrackDataset.from_list([second_track], track_geometry_factory) merged = cast(PandasTrackDataset, first.add_all(second)) for actual, expected in zip(merged.as_list(), expected_dataset.as_list()): @@ -152,7 +169,11 @@ def test_add_two_existing_pandas_datasets(self) -> None: assert merged._geometry_datasets == {} def test_add_all_merge_tracks( - self, first_track: Track, first_track_continuing: Track, second_track: Track + self, + first_track: Track, + first_track_continuing: Track, + second_track: Track, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: ( geometry_dataset_no_offset, @@ -171,7 +192,9 @@ def test_add_all_merge_tracks( ), } dataset = PandasTrackDataset.from_dataframe( - _convert_tracks([first_track_continuing]), geometry_datasets + _convert_tracks([first_track_continuing]), + track_geometry_factory, + geometry_datasets, ) dataset_merged_track = cast( PandasTrackDataset, dataset.add_all([first_track, second_track]) @@ -182,7 +205,7 @@ def test_add_all_merge_tracks( first_track.detections + first_track_continuing.detections, ) expected_dataset = PandasTrackDataset.from_list( - [expected_merged_track, second_track] + [expected_merged_track, second_track], track_geometry_factory ) assert_track_datasets_equal(dataset_merged_track, expected_dataset) assert_track_geometry_dataset_add_all_called_correctly( @@ -203,45 +226,53 @@ def __build_track(self, track_id: str, length: int = 5) -> Track: builder.append_detection() return builder.build_track() - def test_get_by_id(self) -> None: + def test_get_by_id(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - dataset = PandasTrackDataset.from_list([first_track, second_track]) + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) returned = dataset.get_for(first_track.id) assert returned is not None assert_equal_track_properties(returned, first_track) - def test_get_missing(self) -> None: - dataset = PandasTrackDataset() + def test_get_missing(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: + dataset = PandasTrackDataset(track_geometry_factory) returned = dataset.get_for(TrackId("1")) assert returned is None - def test_clear(self) -> None: + def test_clear(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - dataset = PandasTrackDataset.from_list([first_track, second_track]) + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) empty_set = dataset.clear() assert 0 == len(empty_set.as_list()) - def test_remove(self) -> None: + def test_remove(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") tracks_df = _convert_tracks([first_track, second_track]) geometry_dataset, updated_geometry_dataset = create_mock_geometry_dataset() dataset = PandasTrackDataset.from_dataframe( - tracks_df, {RelativeOffsetCoordinate(0, 0): geometry_dataset} + tracks_df, + track_geometry_factory, + {RelativeOffsetCoordinate(0, 0): geometry_dataset}, ) removed_track_set = cast(PandasTrackDataset, dataset.remove(first_track.id)) for actual, expected in zip( removed_track_set.as_list(), - PandasTrackDataset.from_list([second_track]).as_list(), + PandasTrackDataset.from_list( + [second_track], track_geometry_factory + ).as_list(), ): assert_equal_track_properties(actual, expected) geometry_dataset.remove.assert_called_once_with({first_track.id}) @@ -249,12 +280,14 @@ def test_remove(self) -> None: RelativeOffsetCoordinate(0, 0): updated_geometry_dataset, } - def test_len(self) -> None: + def test_len(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") - dataset = PandasTrackDataset.from_list([first_track, second_track]) + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) assert len(dataset) == 2 - empty_dataset = PandasTrackDataset.from_list([]) + empty_dataset = PandasTrackDataset.from_list([], track_geometry_factory) assert len(empty_dataset) == 0 @pytest.mark.parametrize( @@ -282,7 +315,10 @@ def test_split(self, num_tracks: int, batches: int, expected_batches: int) -> No assert_equal_detection_properties(detection, expected_detection) def test_split_with_existing_geometries( - self, first_track: Track, second_track: Track + self, + first_track: Track, + second_track: Track, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: first_track = self.__build_track("1") second_track = self.__build_track("2") @@ -308,13 +344,17 @@ def test_split_with_existing_geometries( } tracks_df = _convert_tracks([first_track, second_track]) - dataset = PandasTrackDataset(tracks_df, geometry_datasets) + dataset = PandasTrackDataset( + track_geometry_factory, tracks_df, geometry_datasets + ) result = cast(list[PythonTrackDataset], dataset.split(batches=2)) assert_track_datasets_equal( - result[0], PandasTrackDataset.from_list([first_track]) + result[0], + PandasTrackDataset.from_list([first_track], track_geometry_factory), ) assert_track_datasets_equal( - result[1], PandasTrackDataset.from_list([second_track]) + result[1], + PandasTrackDataset.from_list([second_track], track_geometry_factory), ) assert geometry_dataset_no_offset.get_for.call_args_list == [ call((first_track.id.id,)), @@ -325,10 +365,14 @@ def test_split_with_existing_geometries( call((second_track.id.id,)), ] - def test_filter_by_minimum_detection_length(self) -> None: + def test_filter_by_minimum_detection_length( + self, track_geometry_factory: TRACK_GEOMETRY_FACTORY + ) -> None: first_track = self.__build_track("1", length=5) second_track = self.__build_track("2", length=10) - dataset = PandasTrackDataset.from_list([first_track, second_track]) + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) filtered_dataset = dataset.filter_by_min_detection_length(7) assert len(filtered_dataset) == 1 diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 73481441c..0a42f48da 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Iterable from unittest.mock import MagicMock, Mock @@ -12,11 +13,15 @@ from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Area, LineSection, Section, SectionId from OTAnalytics.domain.track import ( + TRACK_CLASSIFICATION, + TRACK_GEOMETRY_FACTORY, IntersectionPoint, Track, TrackDataset, TrackGeometryDataset, TrackId, + X, + Y, ) from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( @@ -28,7 +33,10 @@ PygeosTrackGeometryDataset, create_pygeos_track, ) -from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence +from OTAnalytics.plugin_datastore.track_store import ( + PandasByMaxConfidence, + PandasTrackDataset, +) from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser, OttrkParser from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser from tests.conftest import TrackBuilder @@ -203,6 +211,21 @@ def not_intersecting_track() -> Track: return track_builder.build_track() +@pytest.fixture +def single_detection_track_dataset() -> PandasTrackDataset: + data = { + ("Single Detection Track", datetime(2000, 1, 1, 1, 1)): { + X: 1.0, + Y: 3.0, + TRACK_CLASSIFICATION: "car", + }, + } + df = DataFrame.from_dict(data, orient="index") + track_dataset = Mock(spec=PandasTrackDataset) + track_dataset._dataset = df + return track_dataset + + @pytest.fixture def single_detection_track() -> Track: detection = Mock() @@ -640,6 +663,16 @@ def test_get_for_not_existing( expected = create_geometry_dataset_from([], BASE_GEOMETRY) assert_track_geometry_dataset_equals(result, expected) + def test_add_invalid_track( + self, single_detection_track_dataset: PandasTrackDataset + ) -> None: + empty_dataset = PygeosTrackGeometryDataset(BASE_GEOMETRY) + assert not empty_dataset.track_ids + + result = empty_dataset.add_all(single_detection_track_dataset) + assert not result.track_ids + assert result.empty + class TestProfiling: ROUNDS = 1 @@ -647,9 +680,13 @@ class TestProfiling: WARMUP_ROUNDS = 0 @pytest.fixture - def tracks_15min(self, test_data_dir: Path) -> TrackDataset: + def tracks_15min( + self, test_data_dir: Path, track_geometry_factory: TRACK_GEOMETRY_FACTORY + ) -> TrackDataset: ottrk = test_data_dir / "OTCamera19_FR20_2023-05-24_07-00-00.ottrk" - ottrk_parser = OttrkParser(PandasDetectionParser(PandasByMaxConfidence())) + ottrk_parser = OttrkParser( + PandasDetectionParser(PandasByMaxConfidence(), track_geometry_factory) + ) parse_result = ottrk_parser.parse(ottrk) return parse_result.tracks diff --git a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py index cfc3ec505..d41738518 100644 --- a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py @@ -2,7 +2,10 @@ import pytest -from OTAnalytics.domain.track import TrackRepository +from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackRepository +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import ( PandasByMaxConfidence, PandasTrackDataset, @@ -22,6 +25,11 @@ def track_builder_setup_with_sample_data(track_builder: TrackBuilder) -> TrackBu return append_sample_data(track_builder, frame_offset=0, microsecond_offset=0) +@pytest.fixture +def track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: + return PygeosTrackGeometryDataset.from_track_dataset + + def append_sample_data( track_builder: TrackBuilder, frame_offset: int = 0, @@ -59,9 +67,10 @@ def mocked_track_repository() -> Mock: class TestPandasDetectionParser: @pytest.fixture - def parser(self) -> DetectionParser: + def parser(self, track_geometry_factory: TRACK_GEOMETRY_FACTORY) -> DetectionParser: return PandasDetectionParser( PandasByMaxConfidence(), + track_geometry_factory, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, ) @@ -69,6 +78,7 @@ def test_parse_tracks( self, track_builder_setup_with_sample_data: TrackBuilder, parser: DetectionParser, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: detections: list[ dict @@ -84,7 +94,7 @@ def test_parse_tracks( ).as_list() expected_sorted = PandasTrackDataset.from_list( - [track_builder_setup_with_sample_data.build_track()] + [track_builder_setup_with_sample_data.build_track()], track_geometry_factory ).as_list() for sorted, expected in zip(result_sorted_input, expected_sorted): @@ -104,8 +114,11 @@ def test_parse_tracks_consider_minimum_length( mocked_track_repository: Mock, track_builder_setup_with_sample_data: TrackBuilder, track_length_limit: TrackLengthLimit, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: - parser = PandasDetectionParser(PandasByMaxConfidence(), track_length_limit) + parser = PandasDetectionParser( + PandasByMaxConfidence(), track_geometry_factory, track_length_limit + ) detections: list[ dict ] = track_builder_setup_with_sample_data.build_serialized_detections() diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index ca7f92b36..ae3b2bdea 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -25,6 +25,9 @@ ByMaxConfidence, PythonTrackDataset, ) +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import ( PandasByMaxConfidence, PandasTrackDataset, @@ -148,7 +151,9 @@ def python_track_repository() -> TrackRepository: @pytest.fixture def pandas_track_repository() -> TrackRepository: - return TrackRepository(PandasTrackDataset()) + return TrackRepository( + PandasTrackDataset(PygeosTrackGeometryDataset.from_track_dataset) + ) @pytest.fixture @@ -180,7 +185,9 @@ def python_track_parser(python_track_repository: TrackRepository) -> TrackParser @pytest.fixture def pandas_track_parser() -> TrackParser: calculator = PandasByMaxConfidence() - detection_parser = PandasDetectionParser(calculator) + detection_parser = PandasDetectionParser( + calculator, PygeosTrackGeometryDataset.from_track_dataset + ) return OttrkParser(detection_parser) @@ -216,8 +223,14 @@ def python_track_repo_2hours( def pandas_track_repo_15min( track_file_15min: Path, ) -> tuple[TrackRepository, DetectionMetadata]: - track_repository = TrackRepository(PandasTrackDataset()) - track_parser = OttrkParser(PandasDetectionParser(PandasByMaxConfidence())) + track_repository = TrackRepository( + PandasTrackDataset(PygeosTrackGeometryDataset.from_track_dataset) + ) + track_parser = OttrkParser( + PandasDetectionParser( + PandasByMaxConfidence(), PygeosTrackGeometryDataset.from_track_dataset + ) + ) detection_metadata = _fill_track_repository( track_parser, track_repository, [track_file_15min] ) @@ -228,8 +241,14 @@ def pandas_track_repo_15min( def pandas_track_repo_2hours( track_files_2hours: list[Path], ) -> tuple[TrackRepository, DetectionMetadata]: - track_repository = TrackRepository(PandasTrackDataset()) - track_parser = OttrkParser(PandasDetectionParser(PandasByMaxConfidence())) + track_repository = TrackRepository( + PandasTrackDataset(PygeosTrackGeometryDataset.from_track_dataset) + ) + track_parser = OttrkParser( + PandasDetectionParser( + PandasByMaxConfidence(), PygeosTrackGeometryDataset.from_track_dataset + ) + ) detection_metadata = _fill_track_repository( track_parser, track_repository, track_files_2hours ) @@ -356,7 +375,6 @@ def test_2hours( track_repository, _ = pandas_track_repo_2hours section_repository, flow_repository = section_flow_repo_setup use_case = _build_tracks_intersecting_sections(track_repository) - benchmark.pedantic( use_case, args=(section_repository.get_all(),), @@ -424,13 +442,14 @@ def test_15min( create_events = _build_create_events( track_repository, section_repository, event_repository ) - benchmark.pedantic( - create_events, - setup=clear_events, - rounds=self.ROUNDS, - iterations=self.ITERATIONS, - warmup_rounds=self.WARMUP_ROUNDS, - ) + create_events() + # benchmark.pedantic( + # create_events, + # setup=clear_events, + # rounds=self.ROUNDS, + # iterations=self.ITERATIONS, + # warmup_rounds=self.WARMUP_ROUNDS, + # ) def test_2hours( self, diff --git a/tests/conftest.py b/tests/conftest.py index 85d928b37..a60dcbbbc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,9 @@ from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack +from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( + PygeosTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import PandasByMaxConfidence from OTAnalytics.plugin_parser import ottrk_dataformat from OTAnalytics.plugin_parser.otvision_parser import ( @@ -366,7 +369,9 @@ def cyclist_video(test_data_dir: Path) -> Path: def tracks(ottrk_path: Path) -> list[Track]: calculator = PandasByMaxConfidence() detection_parser = PandasDetectionParser( - calculator, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT + calculator, + PygeosTrackGeometryDataset.from_track_dataset, + track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, ) return OttrkParser(detection_parser).parse(ottrk_path).tracks.as_list() # ottrk_parser = OttrkParser( From 3303795a881c3f52eeac85fa35a0b9c9795aa498 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:22:50 +0100 Subject: [PATCH 078/107] Fix benchmarks --- tests/benchmark_otanalytics.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index ae3b2bdea..f7a2c9138 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -442,14 +442,13 @@ def test_15min( create_events = _build_create_events( track_repository, section_repository, event_repository ) - create_events() - # benchmark.pedantic( - # create_events, - # setup=clear_events, - # rounds=self.ROUNDS, - # iterations=self.ITERATIONS, - # warmup_rounds=self.WARMUP_ROUNDS, - # ) + benchmark.pedantic( + create_events, + setup=clear_events, + rounds=self.ROUNDS, + iterations=self.ITERATIONS, + warmup_rounds=self.WARMUP_ROUNDS, + ) def test_2hours( self, From 8ea26dc513b6e5bf3fd4a73e52597342c191d65f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:10:16 +0100 Subject: [PATCH 079/107] Create geometries for offset before splitting TrackDatasets into batches Otherwise, the geometries calculated within the different processes during event creation are lost. --- .../application/use_cases/create_intersection_events.py | 3 +++ OTAnalytics/domain/track.py | 6 ++++++ OTAnalytics/plugin_datastore/python_track_store.py | 7 +++++++ OTAnalytics/plugin_datastore/track_store.py | 7 +++++++ 4 files changed, 23 insertions(+) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 1b4920815..c4d440a41 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -238,6 +238,9 @@ def __init__( def __call__(self, sections: Iterable[Section]) -> list[Event]: filtered_tracks = self._get_tracks.as_dataset() + filtered_tracks.calculate_geometries_for( + {_section.get_offset(EventType.SECTION_ENTER) for _section in sections} + ) batches = filtered_tracks.split(self._intersect_parallelizer.num_processes) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index cd9d2e9cd..d1d862791 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -479,6 +479,12 @@ def filter_by_min_detection_length(self, length: int) -> "TrackDataset": """ raise NotImplementedError + @abstractmethod + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + raise NotImplementedError + class TrackRepository: def __init__(self, dataset: TrackDataset) -> None: diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 741adb078..cdc5ae557 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -404,3 +404,10 @@ def contained_by_sections( ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.contained_by_sections(sections) + + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + for offset in offsets: + if offset not in self._geometry_datasets.keys(): + self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index ecd55e00b..10dafb356 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -353,6 +353,13 @@ def contained_by_sections( geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.contained_by_sections(sections) + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + for offset in offsets: + if offset not in self._geometry_datasets.keys(): + self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) + def _assign_track_classification( data: DataFrame, calculator: PandasTrackClassificationCalculator From 635b35f2738a9ec1666103a8de6d98bd08eefcbf Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Dec 2023 13:58:37 +0100 Subject: [PATCH 080/107] Add vectorized projections calculation --- .../track_geometry_store/pygeos_store.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 3c8f75265..5a0bf912b 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -197,14 +197,15 @@ def __create_entries_from_dataframe( else: new_x = filtered_tracks[track.X] + offset.x * filtered_tracks[track.W] new_y = filtered_tracks[track.Y] + offset.y * filtered_tracks[track.H] - tracks = concat([new_x, new_y], keys=[track.X, track.Y], axis=1).groupby( - level=0, group_keys=True - ) - geometries = tracks.agg(list).apply( + tracks = concat([new_x, new_y], keys=[track.X, track.Y], axis=1) + tracks_by_id = tracks.groupby(level=0, group_keys=True) + geometries = tracks_by_id.agg(list).apply( lambda coords: linestrings(tuple(zip(coords[track.X], coords[track.Y]))), axis=1, ) - projections = tracks.apply(calculate_projection) + # projections = tracks_by_id.apply(calculate_projection) + projections = calculate_all_projections(tracks) + result = concat([geometries, projections], keys=COLUMNS, axis=1) return result @@ -340,10 +341,27 @@ def as_dict(self) -> dict: return self._dataset[COLUMNS].to_dict(orient=ORIENTATION_INDEX) +def calculate_all_projections(tracks: DataFrame) -> DataFrame: + tracks_by_id = tracks.groupby(level=0, group_keys=True) + tracks["last_x"] = tracks_by_id[track.X].shift(1) + tracks["last_y"] = tracks_by_id[track.Y].shift(1) + tracks["length_x"] = tracks[track.X] - tracks["last_x"] + tracks["length_y"] = tracks[track.Y] - tracks["last_y"] + tracks["pow_x"] = tracks["length_x"].pow(2) + tracks["pow_y"] = tracks["length_y"].pow(2) + tracks["sum_x_y_pow"] = tracks["pow_x"] + tracks["pow_y"] + tracks["distance"] = tracks["sum_x_y_pow"].pow(1 / 2) + tracks["distance"].fillna(0, inplace=True) + tracks["cum-distance"] = tracks.groupby(level=0, group_keys=True)[ + "distance" + ].cumsum() + return tracks.groupby(level=0, group_keys=True)["cum-distance"].agg(list) + + def calculate_projection(track_df: DataFrame) -> Series: _track = track_df.reset_index() x_1 = _track.iloc[:-1][track.X].reset_index(drop=True) - y_1 = _track.iloc[:-1][track.X].reset_index(drop=True) + y_1 = _track.iloc[:-1][track.Y].reset_index(drop=True) x_2 = _track.iloc[1:][track.X].reset_index(drop=True) y_2 = _track.iloc[1:][track.Y].reset_index(drop=True) From ae21332f1e6032761fc7a5f99811cdce8b86ce74 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 11:21:22 +0100 Subject: [PATCH 081/107] Fix test to use newest track format --- tests/OTAnalytics/plugin_parser/test_otvision_parser.py | 7 +++++-- tests/conftest.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py index 3d798f49f..f43e66418 100644 --- a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py @@ -277,13 +277,16 @@ def test_parse_ottrk_sample( track_builder_setup_with_sample_data: TrackBuilder, ottrk_parser: OttrkParser, ) -> None: - track_builder_setup_with_sample_data.set_ottrk_version("1.0") + # track_builder_setup_with_sample_data.set_ottrk_version("1.0") ottrk_data = track_builder_setup_with_sample_data.build_ottrk() ottrk_file = test_data_tmp_dir / "sample_file.ottrk" _write_bz2(ottrk_data, ottrk_file) parse_result = ottrk_parser.parse(ottrk_file) - expected_track = track_builder_setup_with_sample_data.build_track() + example_track_builder = TrackBuilder() + example_track_builder.add_track_id("1#1#1") + append_sample_data(example_track_builder) + expected_track = example_track_builder.build_track() expected_detection_classes = frozenset( ["person", "bus", "boat", "truck", "car", "motorcycle", "bicycle", "train"] ) diff --git a/tests/conftest.py b/tests/conftest.py index c5e9f72e7..0283567e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -198,6 +198,8 @@ def get_metadata(self) -> dict: "last_tracked_video_end": self.__to_timestamp( "2020-01-01 00:00:02.950000" ), + "tracking_run_id": "1", + "frame_group": "1", "tracker": { "name": "IOU", "sigma_l": 0.27, From a3c52c80a81fbc8f6404d9b1be2a32a44e7d900b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 11:23:30 +0100 Subject: [PATCH 082/107] Parse multiple ottrk format versions --- tests/OTAnalytics/plugin_parser/test_otvision_parser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py index f43e66418..847e0eadf 100644 --- a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py @@ -271,20 +271,25 @@ def test_parse_whole_ottrk( # TODO What is the expected result? ottrk_parser.parse(ottrk_path) + @pytest.mark.parametrize( + "version,track_id", [("1.0", "legacy#legacy#1"), ("1.1", "1#1#1")] + ) def test_parse_ottrk_sample( self, test_data_tmp_dir: Path, track_builder_setup_with_sample_data: TrackBuilder, ottrk_parser: OttrkParser, + version: str, + track_id: str, ) -> None: - # track_builder_setup_with_sample_data.set_ottrk_version("1.0") + track_builder_setup_with_sample_data.set_ottrk_version(version) ottrk_data = track_builder_setup_with_sample_data.build_ottrk() ottrk_file = test_data_tmp_dir / "sample_file.ottrk" _write_bz2(ottrk_data, ottrk_file) parse_result = ottrk_parser.parse(ottrk_file) example_track_builder = TrackBuilder() - example_track_builder.add_track_id("1#1#1") + example_track_builder.add_track_id(track_id) append_sample_data(example_track_builder) expected_track = example_track_builder.build_track() expected_detection_classes = frozenset( From 4f3e2f33fea2243f268957ad1a1e1b94f8879c0b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 11:26:22 +0100 Subject: [PATCH 083/107] Remove unused code --- .../track_geometry_store/pygeos_store.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 5a0bf912b..2d0ce0fe8 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -3,7 +3,7 @@ from itertools import chain from typing import Any, Iterable, Literal, TypedDict -from pandas import DataFrame, Series, concat +from pandas import DataFrame, concat from pygeos import ( Geometry, contains, @@ -203,7 +203,6 @@ def __create_entries_from_dataframe( lambda coords: linestrings(tuple(zip(coords[track.X], coords[track.Y]))), axis=1, ) - # projections = tracks_by_id.apply(calculate_projection) projections = calculate_all_projections(tracks) result = concat([geometries, projections], keys=COLUMNS, axis=1) @@ -356,16 +355,3 @@ def calculate_all_projections(tracks: DataFrame) -> DataFrame: "distance" ].cumsum() return tracks.groupby(level=0, group_keys=True)["cum-distance"].agg(list) - - -def calculate_projection(track_df: DataFrame) -> Series: - _track = track_df.reset_index() - x_1 = _track.iloc[:-1][track.X].reset_index(drop=True) - y_1 = _track.iloc[:-1][track.Y].reset_index(drop=True) - x_2 = _track.iloc[1:][track.X].reset_index(drop=True) - y_2 = _track.iloc[1:][track.Y].reset_index(drop=True) - - d = ((x_2 - x_1).pow(2) + (y_2 - y_1).pow(2)).pow(1 / 2) - - projection = concat([Series(0), d], ignore_index=True).cumsum() - return projection.agg(list) From b6c61ab9ea96117f558e8e84c9107d3c1b824c12 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 11:35:36 +0100 Subject: [PATCH 084/107] Ignore benchmark files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 629b47b5b..fa9018dff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # OTAnalytics data/ !tests/data +tests/data/OTCamera19_FR20_2023-05-24* tests/scripts/ # Byte-compiled / optimized / DLL files From 921d57017ee07f44203d7c381112514070c0e583 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:12:28 +0100 Subject: [PATCH 085/107] Use dict comprehension --- OTAnalytics/plugin_datastore/python_track_store.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index cdc5ae557..84d443533 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -350,10 +350,10 @@ def _get_geometries_for( self, track_ids: Iterable[TrackId] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: _ids = [track_id.id for track_id in track_ids] - geometry_datasets = {} - for offset, geometry_dataset in self._geometry_datasets.items(): - geometry_datasets[offset] = geometry_dataset.get_for(_ids) - return geometry_datasets + return { + offset: geometry_dataset.get_for(_ids) + for offset, geometry_dataset in self._geometry_datasets.items() + } def __len__(self) -> int: return len(self._tracks) From 37e2f75d34040ef9bc92d627131c2769bc9b9b9a Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:14:36 +0100 Subject: [PATCH 086/107] Remove unused class --- .../plugin_datastore/track_geometry_store/pygeos_store.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 2d0ce0fe8..48dde1c4f 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -1,7 +1,7 @@ from bisect import bisect from collections import defaultdict from itertools import chain -from typing import Any, Iterable, Literal, TypedDict +from typing import Any, Iterable, Literal from pandas import DataFrame, concat from pygeos import ( @@ -87,12 +87,6 @@ def create_pygeos_track( return geometry -class TrackGeometryEntry(TypedDict): - # TODO: Remove if not needed - geometry: Geometry - projection: list[float] - - class InvalidTrackGeometryDataset(Exception): pass From b9f6e11b46859080352b83f1b0e30bd4cd5964db Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:43:41 +0100 Subject: [PATCH 087/107] Remove callable variant to get tracks --- .../application/use_cases/create_events.py | 2 +- .../application/use_cases/track_repository.py | 3 --- .../use_cases/test_create_events.py | 4 +-- .../use_cases/test_track_repository.py | 26 +++++++------------ 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_events.py b/OTAnalytics/application/use_cases/create_events.py index bc3d1f0ec..3ab8651d8 100644 --- a/OTAnalytics/application/use_cases/create_events.py +++ b/OTAnalytics/application/use_cases/create_events.py @@ -110,7 +110,7 @@ def __init__( def __call__(self) -> None: """Create scene enter and leave events and save them to the event repository.""" - tracks = self._get_tracks() + tracks = self._get_tracks.as_list() events = self._scene_action_detector.detect(tracks) self._add_events(events) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 2235993c2..782f1b8c5 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -22,9 +22,6 @@ class GetAllTracks: def __init__(self, track_repository: TrackRepository) -> None: self._track_repository = track_repository - def __call__(self) -> Iterable[Track]: - return self._track_repository.get_all() - def as_list(self) -> list[Track]: return self.as_dataset().as_list() diff --git a/tests/OTAnalytics/application/use_cases/test_create_events.py b/tests/OTAnalytics/application/use_cases/test_create_events.py index 77e9d0b4b..d2d3b2c4b 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_events.py @@ -77,7 +77,7 @@ def test_empty_section_repository_should_not_run_intersection(self) -> None: class TestSimpleCreateSceneEvents: def test_create_scene_events(self, track: Mock, event: Mock) -> None: get_all_tracks = Mock(spec=GetAllTracks) - get_all_tracks.return_value = [track] + get_all_tracks.as_list.return_value = [track] scene_action_detector = Mock(spec=SceneActionDetector) scene_action_detector.detect.return_value = [event] @@ -88,7 +88,7 @@ def test_create_scene_events(self, track: Mock, event: Mock) -> None: ) create_scene_events() - get_all_tracks.assert_called_once() + get_all_tracks.as_list.assert_called_once() scene_action_detector.detect.assert_called_once_with([track]) add_events.assert_called_once_with([event]) diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 97e7d8005..5d909bd9b 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -53,12 +53,6 @@ def track_file_repository(track_files: list[Mock]) -> Mock: class TestGetAllTracks: - def test_get_all_tracks(self, track_repository: Mock, tracks: TrackDataset) -> None: - get_all_tracks = GetAllTracks(track_repository) - result_tracks = get_all_tracks() - assert result_tracks == tracks - track_repository.get_all.assert_called_once() - def test_get_as_dataset(self) -> None: expected_dataset = Mock() track_repository = Mock() @@ -70,20 +64,18 @@ def test_get_as_dataset(self) -> None: track_repository.get_all.assert_called_once() def test_get_as_list(self) -> None: + first_track = Mock() + second_track = Mock() + track_dataset = Mock() + track_dataset.as_list.return_value = [first_track, second_track] track_repository = Mock() + track_repository.get_all.return_value = track_dataset get_tracks = GetAllTracks(track_repository) - with patch.object(GetAllTracks, "as_dataset") as mock_as_dataset: - expected_list = Mock() - filtered_dataset = Mock() - filtered_dataset.as_list.return_value = expected_list - - mock_as_dataset.return_value = filtered_dataset - result = get_tracks.as_list() - - assert result == expected_list - mock_as_dataset.assert_called_once() - filtered_dataset.as_list.assert_called_once() + all_tracks = get_tracks.as_list() + assert all_tracks == [first_track, second_track] + track_repository.get_all.assert_called_once() + track_dataset.as_list.assert_called_once() class TestGetAllTrackIds: From 5eda2ec2e4693c57fc7117621f4da2de0fb6e69f Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:05:54 +0100 Subject: [PATCH 088/107] Update documentation and change class name --- .../use_cases/create_intersection_events.py | 12 ++++-------- .../use_cases/test_create_intersection_events.py | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index c4d440a41..6e789832b 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -18,14 +18,10 @@ from OTAnalytics.domain.types import EventType -class IntersectBySmallestTrackSegments(Intersector): - """ - Implements the intersection strategy by splitting up the track in its smallest - segments and intersecting each of them with the section. - - The smallest segment of a track is to generate a Line with the coordinates of - two neighboring detections in the track. +class IntersectByIntersectionPoints(Intersector): + """Use intersection points of tracks and sections to create events. + This strategy is intended to be used with LineSections. """ def __init__( @@ -253,7 +249,7 @@ def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Ev event_builder = SectionEventBuilder() create_intersection_events = RunCreateIntersectionEvents( - intersect_line_section=IntersectBySmallestTrackSegments(), + intersect_line_section=IntersectByIntersectionPoints(), intersect_area_section=IntersectAreaByTrackPoints(), track_dataset=tracks, sections=sections, diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 290efa818..52bf45f1e 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -5,7 +5,7 @@ from OTAnalytics.application.use_cases.create_intersection_events import ( IntersectAreaByTrackPoints, - IntersectBySmallestTrackSegments, + IntersectByIntersectionPoints, separate_sections, ) from OTAnalytics.domain.geometry import ( @@ -271,9 +271,9 @@ def test_case_line_section_no_intersection(track: Track) -> _TestCase: return _TestCase(track, track_dataset, section, [], []) -class TestIntersectBySmallestTrackSegments: - def _create_intersector(self) -> IntersectBySmallestTrackSegments: - return IntersectBySmallestTrackSegments() +class TestIntersectByIntersectionPoints: + def _create_intersector(self) -> IntersectByIntersectionPoints: + return IntersectByIntersectionPoints() @pytest.mark.parametrize( "test_case_name", From b6fc081e7e2e09b5ae8225eee51a5d4aeac282c3 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:22:58 +0100 Subject: [PATCH 089/107] Raise error if dataframe don't have track id and occurrence set as multi-index --- OTAnalytics/plugin_filter/dataframe_filter.py | 16 ++++++++++----- .../track_visualization/track_viz.py | 7 +++++++ .../plugin_filter/test_dataframe_filter.py | 20 +++++++++++++++++++ .../track_visualization/test_track_viz.py | 9 +++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_filter/dataframe_filter.py b/OTAnalytics/plugin_filter/dataframe_filter.py index 5b1c8937b..1a72d2723 100644 --- a/OTAnalytics/plugin_filter/dataframe_filter.py +++ b/OTAnalytics/plugin_filter/dataframe_filter.py @@ -3,6 +3,7 @@ from pandas import DataFrame, Series +from OTAnalytics.domain import track from OTAnalytics.domain.filter import Conjunction, Filter, FilterBuilder, Predicate @@ -85,9 +86,11 @@ def __init__( self._start_date = start_date def test(self, to_test: DataFrame) -> DataFrame: - # TODO: Only works for DataFrames that have track id and occurrence as - # multi-index - + if not list(to_test.index.names) == [track.TRACK_ID, track.OCCURRENCE]: + raise ValueError( + f"{track.TRACK_ID} and {track.OCCURRENCE} " + "must be index of DataFrame for filtering to worked." + ) return to_test[ to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) >= self._start_date ] @@ -110,8 +113,11 @@ def __init__( self._end_date = end_date def test(self, to_test: DataFrame) -> DataFrame: - # TODO: Only works for DataFrames that have track id and occurrence as - # multi-index + if not list(to_test.index.names) == [track.TRACK_ID, track.OCCURRENCE]: + raise ValueError( + f"{track.TRACK_ID} and {track.OCCURRENCE} " + "must be index of DataFrame for filtering to worked." + ) return to_test[ to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) <= self._end_date ] diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index b5e821bb9..5f91ce883 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -277,6 +277,13 @@ def get_data(self) -> DataFrame: data = self._other.get_data() if data.empty: return data + + if not list(data.index.names) == [track.TRACK_ID, track.OCCURRENCE]: + raise ValueError( + f"{track.TRACK_ID} and {track.OCCURRENCE} " + "must be index of DataFrame for filtering to worked." + ) + ids = [track_id.id for track_id in self._filter.get_ids()] # TODO: This only works for DataFrames with track id and occurrence as # an multi-index. Could not be working with a CachedPandasTrackProvider diff --git a/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py b/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py index 20cfc284a..7bf0ed8c1 100644 --- a/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py +++ b/tests/OTAnalytics/plugin_filter/test_dataframe_filter.py @@ -60,6 +60,26 @@ def track_dataframe(simple_track: Track) -> DataFrame: return convert_tracks_to_dataframe([simple_track]) +class TestDataFrameStartsAtOrAfterDate: + def test_dataframe_with_wrong_index(self) -> None: + df = DataFrame() + df_filter = DataFrameStartsAtOrAfterDate( + OCCURRENCE, datetime(2000, 1, 1, tzinfo=timezone.utc) + ) + with pytest.raises(ValueError): + df_filter.test(df) + + +class TestDataFrameEndsBeforeOrAtDate: + def test_dataframe_with_wrong_index(self) -> None: + df = DataFrame() + df_filter = DataFrameEndsBeforeOrAtDate( + OCCURRENCE, datetime(2000, 1, 1, tzinfo=timezone.utc) + ) + with pytest.raises(ValueError): + df_filter.test(df) + + class TestDataFramePredicates: @pytest.mark.parametrize( "predicate, expected_mask", diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 06e5f9c39..c1e4312d2 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -477,6 +477,15 @@ def test_filter_by_id(self, data_provider: Mock, filter_input: DataFrame) -> Non data_provider.get_data.assert_called_once() id_filter.get_ids.assert_called_once() + def test_filter_by_id_with_no_index_set(self, data_provider: Mock) -> None: + data_provider.get_data.return_value = DataFrame.from_dict( + {1: {"classification": "car", "track_id": "1"}}, orient="index" + ) + id_filter = Mock() + filter_by_id = FilterById(data_provider, id_filter) + with pytest.raises(ValueError): + filter_by_id.get_data() + def test_filter_by_classification( self, filter_input: DataFrame, From 2225ec4e528bc839252a8b5f476c351b1977ec69 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:27:31 +0100 Subject: [PATCH 090/107] Fix typo --- OTAnalytics/plugin_prototypes/track_visualization/track_viz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 5f91ce883..a740e6c01 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -416,7 +416,7 @@ def _convert_tracks(self, tracks: Iterable[Track]) -> DataFrame: def _sort_tracks(self, track_df: DataFrame) -> DataFrame: """Sort the given dataframe by trackId and frame, - if both collumns are available. + if both columns are available. Args: track_df (DataFrame): dataframe of tracks From a8c9b336b617c27c636b8210881eaa69c520a5b8 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 20 Dec 2023 18:50:02 +0100 Subject: [PATCH 091/107] Change TrackDataFrameProvider implementation to use track id and occurrence as multi-index --- .../track_visualization/track_viz.py | 27 ++++++++++--------- .../track_visualization/test_track_viz.py | 3 +-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index a740e6c01..0ea06e9a1 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -412,11 +412,16 @@ def _convert_tracks(self, tracks: Iterable[Track]) -> DataFrame: ] = current_track.classification prepared.append(detection_dict) - return self._sort_tracks(DataFrame(prepared)) + if not prepared: + return DataFrame() + + df = DataFrame(prepared).set_index([track.TRACK_ID, track.OCCURRENCE]) + df.index.names = [track.TRACK_ID, track.OCCURRENCE] + + return self._sort_tracks(df) def _sort_tracks(self, track_df: DataFrame) -> DataFrame: - """Sort the given dataframe by trackId and frame, - if both columns are available. + """Sort the given dataframe by track id and occurrence, Args: track_df (DataFrame): dataframe of tracks @@ -424,10 +429,9 @@ def _sort_tracks(self, track_df: DataFrame) -> DataFrame: Returns: DataFrame: sorted dataframe by track id and frame """ - if (track.TRACK_ID in track_df.columns) and (track.FRAME in track_df.columns): - return track_df.sort_values([track.TRACK_ID, track.FRAME]) - else: + if track_df.empty: return track_df + return track_df.sort_index() class PandasTracksOffsetProvider(PandasDataFrameProvider): @@ -535,9 +539,9 @@ def _reset_cache(self) -> None: def _fetch_new_track_data(self, track_ids: list[TrackId]) -> list[Track]: return [ - track + _track for t_id in track_ids - if (track := self._track_repository.get_for(t_id)) + if (_track := self._track_repository.get_for(t_id)) ] def _cache_without_existing_tracks(self, track_ids: list[TrackId]) -> DataFrame: @@ -557,11 +561,8 @@ def _cache_without_existing_tracks(self, track_ids: list[TrackId]) -> DataFrame: return self._remove_tracks(track_ids) def _remove_tracks(self, track_ids: Iterable[TrackId]) -> DataFrame: - track_id_nums = [t.id for t in track_ids] - cache_without_removed_tracks = self._cache_df.drop( - self._cache_df.index[self._cache_df[track.TRACK_ID].isin(track_id_nums)] - ) - return cache_without_removed_tracks + tracks_to_be_removed = [t.id for t in track_ids] + return self._cache_df.drop(tracks_to_be_removed, axis=0, errors="ignore") def on_tracks_cut(self, cut_tracks_dto: CutTracksDto) -> None: cache_without_cut_tracks = self._remove_tracks(cut_tracks_dto.original_tracks) diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index c1e4312d2..e454ae7d8 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -15,7 +15,6 @@ from OTAnalytics.domain.track import ( OCCURRENCE, TRACK_CLASSIFICATION, - TRACK_ID, Detection, Track, TrackId, @@ -227,7 +226,7 @@ def check_expected_ids( assert provider._cache_df.empty else: - cached_ids = provider._cache_df[TRACK_ID].unique() + cached_ids = provider._cache_df.index.get_level_values(0).unique() expected_detections = sum(len(t.detections) for t in expected_tracks) assert expected_detections == len(provider._cache_df) From 60426c41f6eaeb32a2ad900525f930e85d2eb022 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 09:21:19 +0100 Subject: [PATCH 092/107] Fix error message --- OTAnalytics/plugin_filter/dataframe_filter.py | 4 ++-- .../plugin_prototypes/track_visualization/track_viz.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_filter/dataframe_filter.py b/OTAnalytics/plugin_filter/dataframe_filter.py index 1a72d2723..8202ef5ae 100644 --- a/OTAnalytics/plugin_filter/dataframe_filter.py +++ b/OTAnalytics/plugin_filter/dataframe_filter.py @@ -89,7 +89,7 @@ def test(self, to_test: DataFrame) -> DataFrame: if not list(to_test.index.names) == [track.TRACK_ID, track.OCCURRENCE]: raise ValueError( f"{track.TRACK_ID} and {track.OCCURRENCE} " - "must be index of DataFrame for filtering to worked." + "must be index of DataFrame for filtering to work." ) return to_test[ to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) >= self._start_date @@ -116,7 +116,7 @@ def test(self, to_test: DataFrame) -> DataFrame: if not list(to_test.index.names) == [track.TRACK_ID, track.OCCURRENCE]: raise ValueError( f"{track.TRACK_ID} and {track.OCCURRENCE} " - "must be index of DataFrame for filtering to worked." + "must be index of DataFrame for filtering to work." ) return to_test[ to_test.index.get_level_values(INDEX_LEVEL_OCCURRENCE) <= self._end_date diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 0ea06e9a1..ac0e19831 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -281,7 +281,7 @@ def get_data(self) -> DataFrame: if not list(data.index.names) == [track.TRACK_ID, track.OCCURRENCE]: raise ValueError( f"{track.TRACK_ID} and {track.OCCURRENCE} " - "must be index of DataFrame for filtering to worked." + "must be index of DataFrame for filtering to work." ) ids = [track_id.id for track_id in self._filter.get_ids()] From 8d59daa1c4abb6f95bed5a758436f85a53aeeb18 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 09:36:52 +0100 Subject: [PATCH 093/107] Move iteration over TrackDataset into SceneActionDetector --- OTAnalytics/application/eventlist.py | 8 +++----- OTAnalytics/application/use_cases/create_events.py | 2 +- tests/OTAnalytics/application/test_eventlist.py | 5 +++-- .../application/use_cases/test_create_events.py | 11 ++++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 3c2f0d4de..6ca4d5755 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -1,8 +1,6 @@ -from typing import Iterable - from OTAnalytics.domain.event import Event, EventType, SceneEventBuilder from OTAnalytics.domain.geometry import calculate_direction_vector -from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track import Track, TrackDataset class SceneActionDetector: @@ -62,7 +60,7 @@ def detect_leave_scene(self, track: Track) -> Event: return self._event_builder.create_event(last_detection) - def detect(self, tracks: Iterable[Track]) -> list[Event]: + def detect(self, tracks: TrackDataset) -> list[Event]: """Detect all enter and leave scene events. Args: @@ -72,7 +70,7 @@ def detect(self, tracks: Iterable[Track]) -> list[Event]: Iterable[Event]: the scene events """ events: list[Event] = [] - for track in tracks: + for track in tracks.as_list(): events.append(self.detect_enter_scene(track)) events.append(self.detect_leave_scene(track)) return events diff --git a/OTAnalytics/application/use_cases/create_events.py b/OTAnalytics/application/use_cases/create_events.py index 3ab8651d8..815ffb129 100644 --- a/OTAnalytics/application/use_cases/create_events.py +++ b/OTAnalytics/application/use_cases/create_events.py @@ -110,7 +110,7 @@ def __init__( def __call__(self) -> None: """Create scene enter and leave events and save them to the event repository.""" - tracks = self._get_tracks.as_list() + tracks = self._get_tracks.as_dataset() events = self._scene_action_detector.detect(tracks) self._add_events(events) diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index f3d9eb0d6..e094956ed 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -12,7 +12,7 @@ RelativeOffsetCoordinate, ) from OTAnalytics.domain.section import LineSection, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack @@ -169,7 +169,8 @@ def test_detect( ) -> None: mock_track_1 = Mock(spec=Track) mock_track_2 = Mock(spec=Track) - mock_tracks = [mock_track_1, mock_track_2] + mock_tracks = Mock(spec=TrackDataset) + mock_tracks.as_list.return_value = [mock_track_1, mock_track_2] mock_event_builder = Mock(spec=SceneEventBuilder) scene_action_detector = SceneActionDetector(mock_event_builder) diff --git a/tests/OTAnalytics/application/use_cases/test_create_events.py b/tests/OTAnalytics/application/use_cases/test_create_events.py index d2d3b2c4b..3da21d66b 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_events.py @@ -16,7 +16,7 @@ from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.event import Event from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track import Track, TrackDataset @pytest.fixture @@ -75,9 +75,10 @@ def test_empty_section_repository_should_not_run_intersection(self) -> None: class TestSimpleCreateSceneEvents: - def test_create_scene_events(self, track: Mock, event: Mock) -> None: + def test_create_scene_events(self, event: Mock) -> None: + dataset = Mock(spec=TrackDataset) get_all_tracks = Mock(spec=GetAllTracks) - get_all_tracks.as_list.return_value = [track] + get_all_tracks.as_dataset.return_value = dataset scene_action_detector = Mock(spec=SceneActionDetector) scene_action_detector.detect.return_value = [event] @@ -88,8 +89,8 @@ def test_create_scene_events(self, track: Mock, event: Mock) -> None: ) create_scene_events() - get_all_tracks.as_list.assert_called_once() - scene_action_detector.detect.assert_called_once_with([track]) + get_all_tracks.as_dataset.assert_called_once() + scene_action_detector.detect.assert_called_once_with(dataset) add_events.assert_called_once_with([event]) From 85d416cb14bf9e4ea3ccb8688cdc5060894a6e4e Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 10:13:47 +0100 Subject: [PATCH 094/107] Prepare scene event creation for vectorized methods using a dataset --- OTAnalytics/application/eventlist.py | 45 ++--- .../OTAnalytics/application/test_eventlist.py | 158 ++++++++++-------- 2 files changed, 112 insertions(+), 91 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 6ca4d5755..2ce21c71a 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -1,6 +1,6 @@ from OTAnalytics.domain.event import Event, EventType, SceneEventBuilder from OTAnalytics.domain.geometry import calculate_direction_vector -from OTAnalytics.domain.track import Track, TrackDataset +from OTAnalytics.domain.track import Detection, TrackDataset class SceneActionDetector: @@ -13,7 +13,9 @@ class SceneActionDetector: def __init__(self, scene_event_builder: SceneEventBuilder) -> None: self._event_builder = scene_event_builder - def detect_enter_scene(self, track: Track) -> Event: + def detect_enter_scene( + self, from_detection: Detection, to_detection: Detection, classification: str + ) -> Event: """Detect the first time a road user enters the scene. Args: @@ -23,19 +25,19 @@ def detect_enter_scene(self, track: Track) -> Event: Iterable[Event]: the enter scene event """ self._event_builder.add_event_type(EventType.ENTER_SCENE) - self._event_builder.add_road_user_type(track.classification) - first_detection = track.detections[0] - next_detection = track.detections[1] + self._event_builder.add_road_user_type(classification) self._event_builder.add_direction_vector( calculate_direction_vector( - first_detection.x, first_detection.y, next_detection.x, next_detection.y + from_detection.x, from_detection.y, to_detection.x, to_detection.y ) ) - self._event_builder.add_event_coordinate(first_detection.x, first_detection.y) + self._event_builder.add_event_coordinate(from_detection.x, from_detection.y) - return self._event_builder.create_event(first_detection) + return self._event_builder.create_event(from_detection) - def detect_leave_scene(self, track: Track) -> Event: + def detect_leave_scene( + self, from_detection: Detection, to_detection: Detection, classification: str + ) -> Event: """Detect the last time a road user is seen before leaving the scene. Args: @@ -45,20 +47,15 @@ def detect_leave_scene(self, track: Track) -> Event: Iterable[Event]: the leave scene event """ self._event_builder.add_event_type(EventType.LEAVE_SCENE) - self._event_builder.add_road_user_type(track.classification) - last_detection = track.detections[-1] - second_to_last_detection = track.detections[-2] + self._event_builder.add_road_user_type(classification) self._event_builder.add_direction_vector( calculate_direction_vector( - second_to_last_detection.x, - second_to_last_detection.y, - last_detection.x, - last_detection.y, + from_detection.x, from_detection.y, to_detection.x, to_detection.y ) ) - self._event_builder.add_event_coordinate(last_detection.x, last_detection.y) + self._event_builder.add_event_coordinate(to_detection.x, to_detection.y) - return self._event_builder.create_event(last_detection) + return self._event_builder.create_event(to_detection) def detect(self, tracks: TrackDataset) -> list[Event]: """Detect all enter and leave scene events. @@ -71,6 +68,14 @@ def detect(self, tracks: TrackDataset) -> list[Event]: """ events: list[Event] = [] for track in tracks.as_list(): - events.append(self.detect_enter_scene(track)) - events.append(self.detect_leave_scene(track)) + events.append( + self.detect_enter_scene( + track.detections[0], track.detections[1], track.classification + ) + ) + events.append( + self.detect_leave_scene( + track.detections[-2], track.detections[-1], track.classification + ) + ) return events diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index e094956ed..89a3f6ac8 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -16,93 +16,89 @@ from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack +def create_detection( + _classification: str = "car", + _confidence: float = 0.5, + _x: float = 0.0, + _y: float = 5.0, + _w: float = 15.3, + _h: float = 30.5, + _frame: int = 1, + _occurrence: datetime = datetime(2022, 1, 1, 0, 0, 0, 0), + _interpolated_detection: bool = False, + _track_id: TrackId = TrackId("1"), + _video_name: str = "myhostname_something.mp4", +) -> Detection: + return PythonDetection( + _classification=_classification, + _confidence=_confidence, + _x=_x, + _y=_y, + _w=_w, + _h=_h, + _frame=_frame, + _occurrence=datetime(2022, 1, 1, 0, 0, 0, _frame - 1), + _interpolated_detection=_interpolated_detection, + _track_id=_track_id, + _video_name=_video_name, + ) + + @pytest.fixture def detection() -> Detection: - return PythonDetection( + return create_detection( _classification="car", _confidence=0.5, _x=0.0, _y=5.0, - _w=15.3, - _h=30.5, _frame=1, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 0), - _interpolated_detection=False, _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", ) @pytest.fixture -def track() -> Track: - track_id = TrackId("1") +def track_1() -> Track: + return create_track(1) - detection_1 = PythonDetection( - _classification="car", - _confidence=0.5, + +@pytest.fixture +def track_2() -> Track: + return create_track(2) + + +def create_track(track_number: int) -> Track: + track_id = TrackId(f"{track_number}") + y = 5.0 * track_number + detection_1 = create_detection( _x=0.0, - _y=5.0, - _w=15.3, - _h=30.5, + _y=y, _frame=1, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 0), - _interpolated_detection=False, - _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", + _track_id=track_id, ) - detection_2 = PythonDetection( - _classification="car", - _confidence=0.5, + detection_2 = create_detection( _x=10.0, - _y=5.0, - _w=15.3, - _h=30.5, + _y=y, _frame=2, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 1), - _interpolated_detection=False, - _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", + _track_id=track_id, ) - detection_3 = PythonDetection( - _classification="car", - _confidence=0.5, + detection_3 = create_detection( _x=15.0, - _y=5.0, - _w=15.3, - _h=30.5, + _y=y, _frame=3, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 2), - _interpolated_detection=False, - _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", + _track_id=track_id, ) - detection_4 = PythonDetection( - _classification="car", - _confidence=0.5, + detection_4 = create_detection( _x=20.0, - _y=5.0, - _w=15.3, - _h=30.5, + _y=y, _frame=4, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 3), - _interpolated_detection=False, - _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", + _track_id=track_id, ) - detection_5 = PythonDetection( - _classification="car", - _confidence=0.5, + detection_5 = create_detection( _x=25.0, - _y=5.0, - _w=15.3, - _h=30.5, + _y=y, _frame=5, - _occurrence=datetime(2022, 1, 1, 0, 0, 0, 4), - _interpolated_detection=False, - _track_id=TrackId("1"), - _video_name="myhostname_something.mp4", + _track_id=track_id, ) - return PythonTrack( track_id, "car", @@ -124,12 +120,17 @@ def line_section() -> LineSection: class TestSceneActionDetector: - def test_detect_enter_scene(self, track: Track) -> None: + def test_detect_enter_scene(self, track_1: Track) -> None: + from_detection = track_1.detections[0] + to_detection = track_1.detections[1] + classification = track_1.classification scene_event_builder = SceneEventBuilder() scene_event_builder.add_event_type(EventType.ENTER_SCENE) scene_event_builder.add_road_user_type("car") scene_action_detector = SceneActionDetector(scene_event_builder) - event = scene_action_detector.detect_enter_scene(track) + event = scene_action_detector.detect_enter_scene( + from_detection, to_detection, classification + ) assert event == Event( road_user_id="1", road_user_type="car", @@ -143,12 +144,17 @@ def test_detect_enter_scene(self, track: Track) -> None: video_name="myhostname_something.mp4", ) - def test_detect_leave_scene(self, track: Track) -> None: + def test_detect_leave_scene(self, track_1: Track) -> None: + from_detection = track_1.detections[-2] + to_detection = track_1.detections[-1] + classification = track_1.classification scene_event_builder = SceneEventBuilder() scene_event_builder.add_event_type(EventType.LEAVE_SCENE) scene_event_builder.add_road_user_type("car") scene_action_detector = SceneActionDetector(scene_event_builder) - event = scene_action_detector.detect_leave_scene(track) + event = scene_action_detector.detect_leave_scene( + from_detection, to_detection, classification + ) assert event == Event( road_user_id="1", road_user_type="car", @@ -165,21 +171,31 @@ def test_detect_leave_scene(self, track: Track) -> None: @patch.object(SceneActionDetector, "detect_leave_scene") @patch.object(SceneActionDetector, "detect_enter_scene") def test_detect( - self, mock_detect_enter_scene: Mock, mock_detect_leave_scene: Mock + self, + mock_detect_enter_scene: Mock, + mock_detect_leave_scene: Mock, + track_1: Track, + track_2: Track, ) -> None: - mock_track_1 = Mock(spec=Track) - mock_track_2 = Mock(spec=Track) mock_tracks = Mock(spec=TrackDataset) - mock_tracks.as_list.return_value = [mock_track_1, mock_track_2] + mock_tracks.as_list.return_value = [track_1, track_2] mock_event_builder = Mock(spec=SceneEventBuilder) scene_action_detector = SceneActionDetector(mock_event_builder) scene_action_detector.detect(mock_tracks) - mock_detect_enter_scene.assert_any_call(mock_track_1) - mock_detect_leave_scene.assert_any_call(mock_track_1) - mock_detect_enter_scene.assert_any_call(mock_track_2) - mock_detect_leave_scene.assert_any_call(mock_track_2) + mock_detect_enter_scene.assert_any_call( + track_1.detections[0], track_1.detections[1], track_1.classification + ) + mock_detect_leave_scene.assert_any_call( + track_1.detections[-2], track_1.detections[-1], track_1.classification + ) + mock_detect_enter_scene.assert_any_call( + track_2.detections[0], track_2.detections[1], track_2.classification + ) + mock_detect_leave_scene.assert_any_call( + track_2.detections[-2], track_2.detections[-1], track_2.classification + ) assert mock_detect_enter_scene.call_count == 2 assert mock_detect_leave_scene.call_count == 2 From 62f5105f88f107c90210499d15783cc61310404b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 11:07:11 +0100 Subject: [PATCH 095/107] Move code to consume first and last track segments into TrackDataset --- OTAnalytics/application/eventlist.py | 17 +++--- OTAnalytics/domain/track.py | 12 +++++ .../plugin_datastore/python_track_store.py | 14 ++++- OTAnalytics/plugin_datastore/track_store.py | 28 +++++++++- .../OTAnalytics/application/test_eventlist.py | 23 ++------ .../test_python_track_storage.py | 42 +++++++++++++++ .../plugin_datastore/test_track_store.py | 54 ++++++++++++++++++- 7 files changed, 157 insertions(+), 33 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 2ce21c71a..b8f27a7c3 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -67,15 +67,14 @@ def detect(self, tracks: TrackDataset) -> list[Event]: Iterable[Event]: the scene events """ events: list[Event] = [] - for track in tracks.as_list(): - events.append( - self.detect_enter_scene( - track.detections[0], track.detections[1], track.classification - ) + tracks.apply_to_first_segments( + lambda one, two, three: events.append( + self.detect_enter_scene(one, two, three) ) - events.append( - self.detect_leave_scene( - track.detections[-2], track.detections[-1], track.classification - ) + ) + tracks.apply_to_last_segments( + lambda one, two, three: events.append( + self.detect_leave_scene(one, two, three) ) + ) return events diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index f248d7a63..bd0279a8f 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -488,6 +488,18 @@ def calculate_geometries_for( ) -> None: raise NotImplementedError + @abstractmethod + def apply_to_first_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + raise NotImplementedError + + @abstractmethod + def apply_to_last_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + raise NotImplementedError + class TrackRepository: def __init__(self, dataset: TrackDataset) -> None: diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 84d443533..f272522f7 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Iterable, Optional, Sequence +from typing import Callable, Iterable, Optional, Sequence from more_itertools import batched @@ -411,3 +411,15 @@ def calculate_geometries_for( for offset in offsets: if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) + + def apply_to_first_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + for track in self.as_list(): + consumer(track.detections[0], track.detections[1], track.classification) + + def apply_to_last_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + for track in self.as_list(): + consumer(track.detections[-2], track.detections[-1], track.classification) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 10dafb356..c27fa25dd 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Any, Iterable, Optional, Sequence +from typing import Any, Callable, Iterable, Optional, Sequence import pandas from more_itertools import batched @@ -75,6 +75,18 @@ def track_id(self) -> TrackId: def video_name(self) -> str: return self.__get_attribute(track.VIDEO_NAME) + def __eq__(self, other: Any) -> bool: + if not isinstance(other, PandasDetection): + return False + return ( + self._track_id == other._track_id + and self._occurrence == other._occurrence + and (self._data == other._data).all() + ) + + def __hash__(self) -> int: + return hash(self._track_id) + hash(self._occurrence) + hash(self._data) + @dataclass class PandasTrack(Track): @@ -360,6 +372,20 @@ def calculate_geometries_for( if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) + def apply_to_first_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + for actual in self.as_list(): + consumer(actual.detections[0], actual.detections[1], actual.classification) + + def apply_to_last_segments( + self, consumer: Callable[[Detection, Detection, str], None] + ) -> None: + for actual in self.as_list(): + consumer( + actual.detections[-2], actual.detections[-1], actual.classification + ) + def _assign_track_classification( data: DataFrame, calculator: PandasTrackClassificationCalculator diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index 89a3f6ac8..3ddf3e4c5 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest @@ -168,12 +168,8 @@ def test_detect_leave_scene(self, track_1: Track) -> None: video_name="myhostname_something.mp4", ) - @patch.object(SceneActionDetector, "detect_leave_scene") - @patch.object(SceneActionDetector, "detect_enter_scene") def test_detect( self, - mock_detect_enter_scene: Mock, - mock_detect_leave_scene: Mock, track_1: Track, track_2: Track, ) -> None: @@ -184,18 +180,5 @@ def test_detect( scene_action_detector = SceneActionDetector(mock_event_builder) scene_action_detector.detect(mock_tracks) - mock_detect_enter_scene.assert_any_call( - track_1.detections[0], track_1.detections[1], track_1.classification - ) - mock_detect_leave_scene.assert_any_call( - track_1.detections[-2], track_1.detections[-1], track_1.classification - ) - mock_detect_enter_scene.assert_any_call( - track_2.detections[0], track_2.detections[1], track_2.classification - ) - mock_detect_leave_scene.assert_any_call( - track_2.detections[-2], track_2.detections[-1], track_2.classification - ) - - assert mock_detect_enter_scene.call_count == 2 - assert mock_detect_leave_scene.call_count == 2 + mock_tracks.apply_to_first_segments.assert_called_once() + mock_tracks.apply_to_last_segments.assert_called_once() diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index dcd6a89fb..d235dba76 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -412,3 +412,45 @@ def test_filter_by_minimum_detection_length( filtered_dataset = dataset.filter_by_min_detection_length(3) assert list(filtered_dataset) == [second_track] + + def test_apply_to_first_segments( + self, + first_track: Track, + second_track: Track, + ) -> None: + mock_consumer = Mock() + dataset = PythonTrackDataset.from_list([first_track, second_track]) + + dataset.apply_to_first_segments(mock_consumer) + + mock_consumer.assert_any_call( + first_track.detections[0], + first_track.detections[1], + first_track.classification, + ) + mock_consumer.assert_any_call( + second_track.detections[0], + second_track.detections[1], + second_track.classification, + ) + + def test_apply_to_last_segments( + self, + first_track: Track, + second_track: Track, + ) -> None: + mock_consumer = Mock() + dataset = PythonTrackDataset.from_list([first_track, second_track]) + + dataset.apply_to_last_segments(mock_consumer) + + mock_consumer.assert_any_call( + first_track.detections[-2], + first_track.detections[-1], + first_track.classification, + ) + mock_consumer.assert_any_call( + second_track.detections[-2], + second_track.detections[-1], + second_track.classification, + ) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index e41be3616..1ad38cc59 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -320,8 +320,6 @@ def test_split_with_existing_geometries( second_track: Track, track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: - first_track = self.__build_track("1") - second_track = self.__build_track("2") first_batch_geometries_no_offset = Mock() second_batch_geometries_no_offset = Mock() geometry_dataset_no_offset, _ = create_mock_geometry_dataset( @@ -378,3 +376,55 @@ def test_filter_by_minimum_detection_length( assert len(filtered_dataset) == 1 for actual_track, expected_track in zip(filtered_dataset, [second_track]): assert_equal_track_properties(actual_track, expected_track) + + def test_apply_to_first_segments( + self, + first_track: Track, + second_track: Track, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + ) -> None: + mock_consumer = Mock() + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) + pandas_track_0 = dataset.as_list()[0] + pandas_track_1 = dataset.as_list()[1] + + dataset.apply_to_first_segments(mock_consumer) + + mock_consumer.assert_any_call( + pandas_track_0.detections[0], + pandas_track_0.detections[1], + pandas_track_0.classification, + ) + mock_consumer.assert_any_call( + pandas_track_1.detections[0], + pandas_track_1.detections[1], + pandas_track_1.classification, + ) + + def test_apply_to_last_segments( + self, + first_track: Track, + second_track: Track, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + ) -> None: + mock_consumer = Mock() + dataset = PandasTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) + pandas_track_0 = dataset.as_list()[0] + pandas_track_1 = dataset.as_list()[1] + + dataset.apply_to_last_segments(mock_consumer) + + mock_consumer.assert_any_call( + pandas_track_0.detections[-2], + pandas_track_0.detections[-1], + pandas_track_0.classification, + ) + mock_consumer.assert_any_call( + pandas_track_1.detections[-2], + pandas_track_1.detections[-1], + pandas_track_1.classification, + ) From c205f472be70e5f45375991d03339a02d61f6d88 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 11:16:57 +0100 Subject: [PATCH 096/107] Rename parameters --- OTAnalytics/application/eventlist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index b8f27a7c3..99f2b7945 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -68,13 +68,13 @@ def detect(self, tracks: TrackDataset) -> list[Event]: """ events: list[Event] = [] tracks.apply_to_first_segments( - lambda one, two, three: events.append( - self.detect_enter_scene(one, two, three) + lambda from_detection, to_detection, classification: events.append( + self.detect_enter_scene(from_detection, to_detection, classification) ) ) tracks.apply_to_last_segments( - lambda one, two, three: events.append( - self.detect_leave_scene(one, two, three) + lambda from_detection, to_detection, classification: events.append( + self.detect_leave_scene(from_detection, to_detection, classification) ) ) return events From a856daaf3a71ad8a8c16a06c594e1ae34de0b907 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 21 Dec 2023 11:32:16 +0100 Subject: [PATCH 097/107] Reduce number of generated flyweights to create scene events --- OTAnalytics/plugin_datastore/track_store.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index c27fa25dd..c71841267 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -375,13 +375,23 @@ def calculate_geometries_for( def apply_to_first_segments( self, consumer: Callable[[Detection, Detection, str], None] ) -> None: - for actual in self.as_list(): + track_segments = self._dataset.groupby(track.TRACK_ID).head(2) + + track_ids = list(track_segments.index.get_level_values(LEVEL_TRACK_ID).unique()) + for track_id in track_ids: + track_frame = track_segments.loc[track_id, :] + actual = PandasTrack(track_id, track_frame) consumer(actual.detections[0], actual.detections[1], actual.classification) def apply_to_last_segments( self, consumer: Callable[[Detection, Detection, str], None] ) -> None: - for actual in self.as_list(): + track_segments = self._dataset.groupby(track.TRACK_ID).tail(2) + + track_ids = list(track_segments.index.get_level_values(LEVEL_TRACK_ID).unique()) + for track_id in track_ids: + track_frame = track_segments.loc[track_id, :] + actual = PandasTrack(track_id, track_frame) consumer( actual.detections[-2], actual.detections[-1], actual.classification ) From def5dd58da4f540ce223b264572e12e979923d65 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 22 Dec 2023 10:47:39 +0100 Subject: [PATCH 098/107] Change creation of events --- OTAnalytics/application/eventlist.py | 12 +- OTAnalytics/domain/track.py | 10 +- .../plugin_datastore/python_track_store.py | 67 +++++++++-- OTAnalytics/plugin_datastore/track_store.py | 105 ++++++++++++++---- .../test_python_track_storage.py | 72 ++++++++---- .../plugin_datastore/test_track_store.py | 79 +++++++++---- tests/conftest.py | 8 +- 7 files changed, 257 insertions(+), 96 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 99f2b7945..106ea48a2 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -67,14 +67,6 @@ def detect(self, tracks: TrackDataset) -> list[Event]: Iterable[Event]: the scene events """ events: list[Event] = [] - tracks.apply_to_first_segments( - lambda from_detection, to_detection, classification: events.append( - self.detect_enter_scene(from_detection, to_detection, classification) - ) - ) - tracks.apply_to_last_segments( - lambda from_detection, to_detection, classification: events.append( - self.detect_leave_scene(from_detection, to_detection, classification) - ) - ) + tracks.apply_to_first_segments(events.append) + tracks.apply_to_last_segments(events.append) return events diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index bd0279a8f..7d017f8d2 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Callable, Iterable, Iterator, Optional, Sequence +from typing import Any, Callable, Iterable, Iterator, Optional, Sequence from PIL import Image @@ -489,15 +489,11 @@ def calculate_geometries_for( raise NotImplementedError @abstractmethod - def apply_to_first_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: + def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: raise NotImplementedError @abstractmethod - def apply_to_last_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: + def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index f272522f7..2e72c31c7 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -1,13 +1,18 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Callable, Iterable, Optional, Sequence +from typing import Any, Callable, Iterable, Optional, Sequence from more_itertools import batched from OTAnalytics.application.logger import logger from OTAnalytics.domain.common import DataclassValidation -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.event import Event +from OTAnalytics.domain.geometry import ( + ImageCoordinate, + RelativeOffsetCoordinate, + calculate_direction_vector, +) from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, @@ -20,9 +25,11 @@ TrackHasNoDetectionError, TrackId, ) +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, ) +from OTAnalytics.plugin_datastore.track_store import extract_hostname @dataclass(frozen=True) @@ -412,14 +419,54 @@ def calculate_geometries_for( if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) - def apply_to_first_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: + def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: for track in self.as_list(): - consumer(track.detections[0], track.detections[1], track.classification) + event = self.__create_enter_scene_event(track) + consumer(event) + + def __create_enter_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.first_detection.video_name), + occurrence=track.first_detection.occurrence, + frame_number=track.first_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.first_detection.x, track.first_detection.y + ), + event_type=EventType.ENTER_SCENE, + direction_vector=calculate_direction_vector( + track.first_detection.x, + track.first_detection.y, + track.detections[1].x, + track.detections[1].y, + ), + video_name=track.first_detection.video_name, + ) - def apply_to_last_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: + def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: for track in self.as_list(): - consumer(track.detections[-2], track.detections[-1], track.classification) + event = self.__create_leave_scene_event(track) + consumer(event) + + def __create_leave_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.last_detection.video_name), + occurrence=track.last_detection.occurrence, + frame_number=track.last_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.last_detection.x, track.last_detection.y + ), + event_type=EventType.LEAVE_SCENE, + direction_vector=calculate_direction_vector( + track.detections[-2].x, + track.detections[-2].y, + track.last_detection.x, + track.last_detection.y, + ), + video_name=track.last_detection.video_name, + ) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index c71841267..4efa7e822 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -1,3 +1,4 @@ +import re from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime @@ -9,7 +10,17 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.event import ( + FILE_NAME_PATTERN, + HOSTNAME, + Event, + ImproperFormattedFilename, +) +from OTAnalytics.domain.geometry import ( + ImageCoordinate, + RelativeOffsetCoordinate, + calculate_direction_vector, +) from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, @@ -20,6 +31,7 @@ TrackGeometryDataset, TrackId, ) +from OTAnalytics.domain.types import EventType class PandasDetection(Detection): @@ -160,6 +172,29 @@ def calculate(self, detections: DataFrame) -> DataFrame: LEVEL_OCCURRENCE = 1 +def extract_hostname(name: str) -> str: + """Extract hostname from name. + + Args: + name (Path): name containing the hostname. + + Raises: + InproperFormattedFilename: if the name is not formatted as expected, an + exception will be raised. + + Returns: + str: the hostname. + """ + match = re.search( + FILE_NAME_PATTERN, + name, + ) + if match: + hostname: str = match.group(HOSTNAME) + return hostname + raise ImproperFormattedFilename(f"Could not parse {name}. Hostname is missing.") + + class PandasTrackDataset(TrackDataset): def __init__( self, @@ -372,29 +407,55 @@ def calculate_geometries_for( if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) - def apply_to_first_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: - track_segments = self._dataset.groupby(track.TRACK_ID).head(2) - - track_ids = list(track_segments.index.get_level_values(LEVEL_TRACK_ID).unique()) - for track_id in track_ids: - track_frame = track_segments.loc[track_id, :] - actual = PandasTrack(track_id, track_frame) - consumer(actual.detections[0], actual.detections[1], actual.classification) + def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: + first_detections = self._dataset.groupby(level=0, group_keys=True) + self._dataset["next_x"] = first_detections[track.X].shift(-1) + self._dataset["next_y"] = first_detections[track.Y].shift(-1) + first_segments: DataFrame = ( + self._dataset.groupby(level=0, group_keys=True).head(1).copy() + ) - def apply_to_last_segments( - self, consumer: Callable[[Detection, Detection, str], None] - ) -> None: - track_segments = self._dataset.groupby(track.TRACK_ID).tail(2) - - track_ids = list(track_segments.index.get_level_values(LEVEL_TRACK_ID).unique()) - for track_id in track_ids: - track_frame = track_segments.loc[track_id, :] - actual = PandasTrack(track_id, track_frame) - consumer( - actual.detections[-2], actual.detections[-1], actual.classification + for index, row in first_segments.iterrows(): + event = Event( + road_user_id=index[0], + road_user_type=row[track.TRACK_CLASSIFICATION], + hostname=extract_hostname(row[track.VIDEO_NAME]), + occurrence=index[1].to_pydatetime(), + frame_number=row[track.FRAME], + section_id=None, + event_coordinate=ImageCoordinate(row[track.X], row[track.Y]), + event_type=EventType.ENTER_SCENE, + direction_vector=calculate_direction_vector( + row[track.X], row[track.Y], row["next_x"], row["next_y"] + ), + video_name=row[track.VIDEO_NAME], + ) + consumer(event) + + def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: + first_detections = self._dataset.groupby(level=0, group_keys=True) + self._dataset["previous_x"] = first_detections[track.X].shift(1) + self._dataset["previous_y"] = first_detections[track.Y].shift(1) + first_segments: DataFrame = self._dataset.groupby( + level=0, group_keys=True + ).tail(1) + + for index, row in first_segments.iterrows(): + event = Event( + road_user_id=index[0], + road_user_type=row[track.TRACK_CLASSIFICATION], + hostname=extract_hostname(row[track.VIDEO_NAME]), + occurrence=index[1], + frame_number=row[track.FRAME], + section_id=None, + event_coordinate=ImageCoordinate(row[track.X], row[track.Y]), + event_type=EventType.LEAVE_SCENE, + direction_vector=calculate_direction_vector( + row["previous_x"], row["previous_y"], row[track.X], row[track.Y] + ), + video_name=row[track.VIDEO_NAME], ) + consumer(event) def _assign_track_classification( diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index d235dba76..d867a730a 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -5,8 +5,12 @@ import pytest -from OTAnalytics.domain.event import VIDEO_NAME -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.event import VIDEO_NAME, Event +from OTAnalytics.domain.geometry import ( + ImageCoordinate, + RelativeOffsetCoordinate, + calculate_direction_vector, +) from OTAnalytics.domain.track import ( Detection, Track, @@ -14,12 +18,14 @@ TrackHasNoDetectionError, TrackId, ) +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonDetection, PythonTrack, PythonTrackDataset, ) +from OTAnalytics.plugin_datastore.track_store import extract_hostname from OTAnalytics.plugin_parser import ottrk_dataformat as ottrk_format from tests.conftest import TrackBuilder from tests.OTAnalytics.plugin_datastore.conftest import ( @@ -423,16 +429,8 @@ def test_apply_to_first_segments( dataset.apply_to_first_segments(mock_consumer) - mock_consumer.assert_any_call( - first_track.detections[0], - first_track.detections[1], - first_track.classification, - ) - mock_consumer.assert_any_call( - second_track.detections[0], - second_track.detections[1], - second_track.classification, - ) + mock_consumer.assert_any_call(self.__create_enter_scene_event(first_track)) + mock_consumer.assert_any_call(self.__create_enter_scene_event(second_track)) def test_apply_to_last_segments( self, @@ -444,13 +442,47 @@ def test_apply_to_last_segments( dataset.apply_to_last_segments(mock_consumer) - mock_consumer.assert_any_call( - first_track.detections[-2], - first_track.detections[-1], - first_track.classification, + mock_consumer.assert_any_call(self.__create_leave_scene_event(first_track)) + mock_consumer.assert_any_call(self.__create_leave_scene_event(second_track)) + + def __create_enter_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.first_detection.video_name), + occurrence=track.first_detection.occurrence, + frame_number=track.first_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.first_detection.x, track.first_detection.y + ), + event_type=EventType.ENTER_SCENE, + direction_vector=calculate_direction_vector( + track.first_detection.x, + track.first_detection.y, + track.detections[1].x, + track.detections[1].y, + ), + video_name=track.first_detection.video_name, ) - mock_consumer.assert_any_call( - second_track.detections[-2], - second_track.detections[-1], - second_track.classification, + + def __create_leave_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.last_detection.video_name), + occurrence=track.last_detection.occurrence, + frame_number=track.last_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.last_detection.x, track.last_detection.y + ), + event_type=EventType.LEAVE_SCENE, + direction_vector=calculate_direction_vector( + track.detections[-2].x, + track.detections[-2].y, + track.last_detection.x, + track.last_detection.y, + ), + video_name=track.last_detection.video_name, ) diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 1ad38cc59..29892354b 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -5,7 +5,12 @@ from pandas import DataFrame, Series from OTAnalytics.domain import track -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.event import Event +from OTAnalytics.domain.geometry import ( + ImageCoordinate, + RelativeOffsetCoordinate, + calculate_direction_vector, +) from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, Track, @@ -13,6 +18,7 @@ TrackGeometryDataset, TrackId, ) +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, PythonTrackDataset, @@ -25,6 +31,7 @@ PandasTrack, PandasTrackDataset, _convert_tracks, + extract_hostname, ) from tests.conftest import ( TrackBuilder, @@ -384,23 +391,36 @@ def test_apply_to_first_segments( track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: mock_consumer = Mock() + event_1 = self.__create_enter_scene_event(first_track) + event_2 = self.__create_enter_scene_event(second_track) dataset = PandasTrackDataset.from_list( [first_track, second_track], track_geometry_factory ) - pandas_track_0 = dataset.as_list()[0] - pandas_track_1 = dataset.as_list()[1] dataset.apply_to_first_segments(mock_consumer) - mock_consumer.assert_any_call( - pandas_track_0.detections[0], - pandas_track_0.detections[1], - pandas_track_0.classification, - ) - mock_consumer.assert_any_call( - pandas_track_1.detections[0], - pandas_track_1.detections[1], - pandas_track_1.classification, + mock_consumer.assert_any_call(event_1) + mock_consumer.assert_any_call(event_2) + + def __create_enter_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.first_detection.video_name), + occurrence=track.first_detection.occurrence, + frame_number=track.first_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.first_detection.x, track.first_detection.y + ), + event_type=EventType.ENTER_SCENE, + direction_vector=calculate_direction_vector( + track.first_detection.x, + track.first_detection.y, + track.detections[1].x, + track.detections[1].y, + ), + video_name=track.first_detection.video_name, ) def test_apply_to_last_segments( @@ -410,21 +430,34 @@ def test_apply_to_last_segments( track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: mock_consumer = Mock() + event_1 = self.__create_leave_scene_event(first_track) + event_2 = self.__create_leave_scene_event(second_track) dataset = PandasTrackDataset.from_list( [first_track, second_track], track_geometry_factory ) - pandas_track_0 = dataset.as_list()[0] - pandas_track_1 = dataset.as_list()[1] dataset.apply_to_last_segments(mock_consumer) - mock_consumer.assert_any_call( - pandas_track_0.detections[-2], - pandas_track_0.detections[-1], - pandas_track_0.classification, - ) - mock_consumer.assert_any_call( - pandas_track_1.detections[-2], - pandas_track_1.detections[-1], - pandas_track_1.classification, + mock_consumer.assert_any_call(event_1) + mock_consumer.assert_any_call(event_2) + + def __create_leave_scene_event(self, track: Track) -> Event: + return Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=extract_hostname(track.last_detection.video_name), + occurrence=track.last_detection.occurrence, + frame_number=track.last_detection.frame, + section_id=None, + event_coordinate=ImageCoordinate( + track.last_detection.x, track.last_detection.y + ), + event_type=EventType.LEAVE_SCENE, + direction_vector=calculate_direction_vector( + track.detections[-2].x, + track.detections[-2].y, + track.last_detection.x, + track.last_detection.y, + ), + video_name=track.last_detection.video_name, ) diff --git a/tests/conftest.py b/tests/conftest.py index 0283567e7..b665bac18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,10 +47,10 @@ class TrackBuilder: track_class: str = "car" detection_class: str = "car" confidence: float = 0.5 - x: float = 0 - y: float = 0 - w: float = 10 - h: float = 10 + x: float = 0.0 + y: float = 0.0 + w: float = 10.0 + h: float = 10.0 frame: int = 1 occurrence_year: int = DEFAULT_OCCURRENCE_YEAR occurrence_month: int = DEFAULT_OCCURRENCE_MONTH From 27b981409105587a549b75232e2e94338976591b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 22 Dec 2023 23:14:38 +0100 Subject: [PATCH 099/107] Remove iterrows over dataframe --- .../use_cases/create_intersection_events.py | 6 +-- OTAnalytics/domain/track.py | 4 ++ .../plugin_datastore/python_track_store.py | 3 ++ OTAnalytics/plugin_datastore/track_store.py | 49 ++++++++++++------- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 6e789832b..37d4dc2e2 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -67,11 +67,11 @@ def __do_intersect( event_builder.add_road_user_type(track.classification) for section_id, intersection_point in intersection_points: event_builder.add_section_id(section_id) - detection = track.detections[intersection_point.index] + detection = track.get_detection(intersection_point.index) current_coord = detection.get_coordinate(offset) - prev_coord = track.detections[ + prev_coord = track.get_detection( intersection_point.index - 1 - ].get_coordinate(offset) + ).get_coordinate(offset) direction_vector = self._calculate_direction_vector( prev_coord.x, prev_coord.y, diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 7d017f8d2..412abe6a4 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -214,6 +214,10 @@ def classification(self) -> str: def detections(self) -> list[Detection]: raise NotImplementedError + @abstractmethod + def get_detection(self, index: int) -> Detection: + raise NotImplementedError + @property @abstractmethod def first_detection(self) -> Detection: diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 2e72c31c7..6bfc42155 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -174,6 +174,9 @@ def classification(self) -> str: def detections(self) -> list[Detection]: return self._detections + def get_detection(self, index: int) -> Detection: + return self._detections[index] + def _validate(self) -> None: self._validate_track_has_detections() self._validate_detections_sorted_by_occurrence() diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 4efa7e822..aae23a742 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -117,6 +117,9 @@ def classification(self) -> str: def detections(self) -> list[Detection]: return [PandasDetection(self._id, row) for _, row in self._data.iterrows()] + def get_detection(self, index: int) -> Detection: + return PandasDetection(self._id, self._data.iloc[index]) + @property def first_detection(self) -> Detection: return PandasDetection(self._id, self._data.iloc[0]) @@ -415,20 +418,25 @@ def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: self._dataset.groupby(level=0, group_keys=True).head(1).copy() ) - for index, row in first_segments.iterrows(): + as_dict = first_segments.reset_index().to_dict("index") + + for value in as_dict.values(): event = Event( - road_user_id=index[0], - road_user_type=row[track.TRACK_CLASSIFICATION], - hostname=extract_hostname(row[track.VIDEO_NAME]), - occurrence=index[1].to_pydatetime(), - frame_number=row[track.FRAME], + road_user_id=value[track.TRACK_ID], + road_user_type=value[track.TRACK_CLASSIFICATION], + hostname=extract_hostname(value[track.VIDEO_NAME]), + occurrence=value[track.OCCURRENCE].to_pydatetime(), + frame_number=value[track.FRAME], section_id=None, - event_coordinate=ImageCoordinate(row[track.X], row[track.Y]), + event_coordinate=ImageCoordinate(value[track.X], value[track.Y]), event_type=EventType.ENTER_SCENE, direction_vector=calculate_direction_vector( - row[track.X], row[track.Y], row["next_x"], row["next_y"] + value[track.X], + value[track.Y], + value["next_x"], + value["next_y"], ), - video_name=row[track.VIDEO_NAME], + video_name=value[track.VIDEO_NAME], ) consumer(event) @@ -440,20 +448,25 @@ def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: level=0, group_keys=True ).tail(1) - for index, row in first_segments.iterrows(): + as_dict = first_segments.reset_index().to_dict("index") + + for value in as_dict.values(): event = Event( - road_user_id=index[0], - road_user_type=row[track.TRACK_CLASSIFICATION], - hostname=extract_hostname(row[track.VIDEO_NAME]), - occurrence=index[1], - frame_number=row[track.FRAME], + road_user_id=value[track.TRACK_ID], + road_user_type=value[track.TRACK_CLASSIFICATION], + hostname=extract_hostname(value[track.VIDEO_NAME]), + occurrence=value[track.OCCURRENCE].to_pydatetime(), + frame_number=value[track.FRAME], section_id=None, - event_coordinate=ImageCoordinate(row[track.X], row[track.Y]), + event_coordinate=ImageCoordinate(value[track.X], value[track.Y]), event_type=EventType.LEAVE_SCENE, direction_vector=calculate_direction_vector( - row["previous_x"], row["previous_y"], row[track.X], row[track.Y] + value["previous_x"], + value["previous_y"], + value[track.X], + value[track.Y], ), - video_name=row[track.VIDEO_NAME], + video_name=value[track.VIDEO_NAME], ) consumer(event) From 32eaecc0fbb692b78fc5fc711dabd1d2e8d2dd28 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 08:57:21 +0100 Subject: [PATCH 100/107] Remove unnecessary mock --- tests/OTAnalytics/application/test_eventlist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index 3ddf3e4c5..751e1a792 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -174,7 +174,6 @@ def test_detect( track_2: Track, ) -> None: mock_tracks = Mock(spec=TrackDataset) - mock_tracks.as_list.return_value = [track_1, track_2] mock_event_builder = Mock(spec=SceneEventBuilder) scene_action_detector = SceneActionDetector(mock_event_builder) From 6e2c13e9368eac82fb701427acb137134a9159b7 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 09:10:50 +0100 Subject: [PATCH 101/107] Move TrackRepository into separate module --- .../application/analysis/traffic_counting.py | 3 +- OTAnalytics/application/datastore.py | 6 +- OTAnalytics/application/state.py | 6 +- .../application/use_cases/event_repository.py | 2 +- .../use_cases/highlight_intersections.py | 3 +- .../use_cases/intersection_repository.py | 2 +- .../application/use_cases/load_track_files.py | 2 +- .../application/use_cases/track_repository.py | 6 +- OTAnalytics/domain/track.py | 234 ----------------- OTAnalytics/domain/track_repository.py | 239 ++++++++++++++++++ OTAnalytics/plugin_parser/otvision_parser.py | 2 +- .../track_visualization/track_viz.py | 2 + OTAnalytics/plugin_ui/cli.py | 2 +- .../customtkinter_gui/dummy_viewmodel.py | 3 +- OTAnalytics/plugin_ui/main_application.py | 2 +- .../plugin_ui/visualization/visualization.py | 3 +- .../analysis/test_traffic_counting.py | 3 +- .../OTAnalytics/application/test_datastore.py | 3 +- tests/OTAnalytics/application/test_state.py | 9 +- .../use_cases/test_highlight_intersections.py | 9 +- .../use_cases/test_track_repository.py | 9 +- tests/OTAnalytics/domain/test_track.py | 6 +- .../plugin_parser/test_otvision_parser.py | 2 +- .../plugin_parser/test_pandas_parser.py | 3 +- .../track_visualization/test_track_viz.py | 3 +- tests/OTAnalytics/plugin_ui/test_cli.py | 3 +- tests/benchmark_otanalytics.py | 2 +- 27 files changed, 281 insertions(+), 288 deletions(-) create mode 100644 OTAnalytics/domain/track_repository.py diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index 1881ea512..c1cebe4c6 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -16,7 +16,8 @@ from OTAnalytics.domain.event import Event, EventRepository from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import TrackId, TrackRepository +from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.types import EventType LEVEL_FROM_SECTION = "from section" diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index cbf838a4c..98feef438 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -22,11 +22,9 @@ SectionListObserver, SectionRepository, ) -from OTAnalytics.domain.track import ( - TrackDataset, +from OTAnalytics.domain.track import TrackDataset, TrackId, TrackImage +from OTAnalytics.domain.track_repository import ( TrackFileRepository, - TrackId, - TrackImage, TrackListObserver, TrackRepository, ) diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index 4aebfebbc..5cdff1bde 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -17,10 +17,8 @@ SectionRepositoryEvent, SectionType, ) -from OTAnalytics.domain.track import ( - Detection, - TrackId, - TrackImage, +from OTAnalytics.domain.track import Detection, TrackId, TrackImage +from OTAnalytics.domain.track_repository import ( TrackListObserver, TrackObserver, TrackRepository, diff --git a/OTAnalytics/application/use_cases/event_repository.py b/OTAnalytics/application/use_cases/event_repository.py index a0b3544e5..e3006123e 100644 --- a/OTAnalytics/application/use_cases/event_repository.py +++ b/OTAnalytics/application/use_cases/event_repository.py @@ -7,7 +7,7 @@ SectionListObserver, SectionRepositoryEvent, ) -from OTAnalytics.domain.track import TrackListObserver, TrackRepositoryEvent +from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent class AddEvents: diff --git a/OTAnalytics/application/use_cases/highlight_intersections.py b/OTAnalytics/application/use_cases/highlight_intersections.py index 374226674..93fa95dc1 100644 --- a/OTAnalytics/application/use_cases/highlight_intersections.py +++ b/OTAnalytics/application/use_cases/highlight_intersections.py @@ -9,7 +9,8 @@ from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider, TrackRepository +from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track_repository import TrackRepository class IntersectionRepository(ABC): diff --git a/OTAnalytics/application/use_cases/intersection_repository.py b/OTAnalytics/application/use_cases/intersection_repository.py index f288f35e4..a120cabe6 100644 --- a/OTAnalytics/application/use_cases/intersection_repository.py +++ b/OTAnalytics/application/use_cases/intersection_repository.py @@ -8,7 +8,7 @@ SectionListObserver, SectionRepositoryEvent, ) -from OTAnalytics.domain.track import TrackListObserver, TrackRepositoryEvent +from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent class ClearAllIntersections(SectionListObserver, TrackListObserver): diff --git a/OTAnalytics/application/use_cases/load_track_files.py b/OTAnalytics/application/use_cases/load_track_files.py index 296d473b7..d4e9243b0 100644 --- a/OTAnalytics/application/use_cases/load_track_files.py +++ b/OTAnalytics/application/use_cases/load_track_files.py @@ -7,7 +7,7 @@ ) from OTAnalytics.application.state import TracksMetadata, VideosMetadata from OTAnalytics.domain.progress import ProgressbarBuilder -from OTAnalytics.domain.track import TrackFileRepository, TrackRepository +from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 782f1b8c5..2672f402d 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -2,12 +2,10 @@ from typing import Iterable from OTAnalytics.application.logger import logger -from OTAnalytics.domain.track import ( +from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track_repository import ( RemoveMultipleTracksError, - Track, - TrackDataset, TrackFileRepository, - TrackId, TrackRepository, ) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 412abe6a4..713af6667 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -1,14 +1,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime -from pathlib import Path from typing import Any, Callable, Iterable, Iterator, Optional, Sequence from PIL import Image from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.observer import Subject from OTAnalytics.domain.section import Section, SectionId MIN_NUMBER_OF_DETECTIONS = 5 @@ -34,71 +32,6 @@ def __str__(self) -> str: return self.id -@dataclass(frozen=True) -class TrackRepositoryEvent: - added: list[TrackId] - removed: list[TrackId] - - -class TrackListObserver(ABC): - """ - Interface to listen to changes to a list of tracks. - """ - - @abstractmethod - def notify_tracks(self, track_event: TrackRepositoryEvent) -> None: - """ - Notifies that the given tracks have been added. - - Args: - track_event (TrackRepositoryEvent): list of added or removed tracks. - """ - raise NotImplementedError - - -class TrackObserver(ABC): - """ - Interface to listen to changes of a single track. - """ - - @abstractmethod - def notify_track(self, track_id: Optional[TrackId]) -> None: - """ - Notifies that the track of the given id has changed. - - Args: - track_id (Optional[TrackId]): id of the changed track - """ - pass - - -class TrackSubject: - """ - Helper class to handle and notify observers - """ - - def __init__(self) -> None: - self.observers: set[TrackObserver] = set() - - def register(self, observer: TrackObserver) -> None: - """ - Listen to events. - - Args: - observer (TrackObserver): listener to add - """ - self.observers.add(observer) - - def notify(self, track_id: Optional[TrackId]) -> None: - """ - Notifies observers about the track id. - - Args: - track_id (Optional[TrackId]): id of the changed track - """ - [observer.notify_track(track_id) for observer in self.observers] - - class TrackError(Exception): def __init__(self, track_id: TrackId, *args: object) -> None: super().__init__(*args) @@ -347,31 +280,6 @@ def calculate(self, detections: list[Detection]) -> str: raise NotImplementedError -class TrackRemoveError(Exception): - def __init__(self, track_id: TrackId, message: str) -> None: - """Exception to be raised if track can not be removed. - - Args: - track_id (TrackId): the track id of the track to be removed. - message (str): the error message. - """ - super().__init__(message) - self._track_id = track_id - - -class RemoveMultipleTracksError(Exception): - """Exception to be raised if multiple tracks can not be removed. - - Args: - track_ids (list[TrackId]): the track id of the track to be removed. - message (str): the error message. - """ - - def __init__(self, track_ids: list[TrackId], message: str): - super().__init__(message) - self._track_ids = track_ids - - @dataclass class IntersectionPoint: index: int @@ -501,148 +409,6 @@ def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: raise NotImplementedError -class TrackRepository: - def __init__(self, dataset: TrackDataset) -> None: - self._dataset = dataset - self.observers = Subject[TrackRepositoryEvent]() - - def register_tracks_observer(self, observer: TrackListObserver) -> None: - """ - Listen to changes of the repository. - - Args: - observer (TrackListObserver): listener to be notified about changes - """ - self.observers.register(observer.notify_tracks) - - def add_all(self, tracks: Iterable[Track]) -> None: - """ - Add multiple tracks to the repository and notify only once about it. - - Args: - tracks (Iterable[Track]): tracks to be added - """ - self._dataset = self._dataset.add_all(tracks) - new_tracks = [track.id for track in tracks] - if new_tracks: - self.observers.notify(TrackRepositoryEvent(new_tracks, [])) - - def get_for(self, id: TrackId) -> Optional[Track]: - """ - Retrieve a track for the given id. - - Args: - id (TrackId): id to search for - - Returns: - Optional[Track]: track if it exists - """ - return self._dataset.get_for(id) - - def get_all(self) -> TrackDataset: - """ - Retrieve all tracks. - - Returns: - list[Track]: all tracks within the repository - """ - return self._dataset - - def get_all_ids(self) -> Iterable[TrackId]: - """Get all track ids in this repository. - - Returns: - Iterable[TrackId]: the track ids. - """ - return self._dataset.get_all_ids() - - def remove(self, track_id: TrackId) -> None: - """Remove track by its id and notify observers - - Raises: - TrackRemoveError: if track does not exist in repository. - - Args: - track_id (TrackId): the id of the track to be removed. - """ - try: - self._dataset = self._dataset.remove(track_id) - except KeyError: - raise TrackRemoveError( - track_id, f"Trying to remove non existing track with id '{track_id.id}'" - ) - # TODO: Pass removed track id to notify when moving observers to - # application layer - self.observers.notify(TrackRepositoryEvent([], [track_id])) - - def remove_multiple(self, track_ids: set[TrackId]) -> None: - failed_tracks: list[TrackId] = [] - for track_id in track_ids: - try: - self._dataset = self._dataset.remove(track_id) - except KeyError: - failed_tracks.append(track_id) - # TODO: Pass removed track id to notify when moving observers to - # application layer - - if failed_tracks: - raise RemoveMultipleTracksError( - failed_tracks, - ( - "Multiple tracks with following ids could not be removed." - f" '{[failed_track.id for failed_track in failed_tracks]}'" - ), - ) - self.observers.notify(TrackRepositoryEvent([], list(track_ids))) - - def split(self, chunks: int) -> Iterable[TrackDataset]: - return self._dataset.split(chunks) - - def clear(self) -> None: - """ - Clear the repository and inform the observers about the empty repository. - """ - removed = list(self._dataset.get_all_ids()) - self._dataset = self._dataset.clear() - self.observers.notify(TrackRepositoryEvent([], removed)) - - def __len__(self) -> int: - return len(self._dataset) - - -class TrackFileRepository: - def __init__(self) -> None: - self._files: set[Path] = set() - - def add(self, file: Path) -> None: - """ - Add a single track file the repository. - - Args: - file (Path): track file to be added. - """ - self._files.add(file) - - def add_all(self, files: Iterable[Path]) -> None: - """ - Add multiple files to the repository. - - Args: - files (Iterable[Path]): the files to be added. - """ - for file in files: - self.add(file) - - def get_all(self) -> set[Path]: - """ - Retrieve all track files. - - Returns: - set[Path]: all tracks within the repository. - """ - return self._files.copy() - - class TrackIdProvider(ABC): """Interface to provide track ids.""" diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py new file mode 100644 index 000000000..5af83973f --- /dev/null +++ b/OTAnalytics/domain/track_repository.py @@ -0,0 +1,239 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional + +from OTAnalytics.domain.observer import Subject +from OTAnalytics.domain.track import Track, TrackDataset, TrackId + + +@dataclass(frozen=True) +class TrackRepositoryEvent: + added: list[TrackId] + removed: list[TrackId] + + +class TrackListObserver(ABC): + """ + Interface to listen to changes to a list of tracks. + """ + + @abstractmethod + def notify_tracks(self, track_event: TrackRepositoryEvent) -> None: + """ + Notifies that the given tracks have been added. + + Args: + track_event (TrackRepositoryEvent): list of added or removed tracks. + """ + raise NotImplementedError + + +class TrackObserver(ABC): + """ + Interface to listen to changes of a single track. + """ + + @abstractmethod + def notify_track(self, track_id: Optional[TrackId]) -> None: + """ + Notifies that the track of the given id has changed. + + Args: + track_id (Optional[TrackId]): id of the changed track + """ + pass + + +class TrackSubject: + """ + Helper class to handle and notify observers + """ + + def __init__(self) -> None: + self.observers: set[TrackObserver] = set() + + def register(self, observer: TrackObserver) -> None: + """ + Listen to events. + + Args: + observer (TrackObserver): listener to add + """ + self.observers.add(observer) + + def notify(self, track_id: Optional[TrackId]) -> None: + """ + Notifies observers about the track id. + + Args: + track_id (Optional[TrackId]): id of the changed track + """ + [observer.notify_track(track_id) for observer in self.observers] + + +class TrackRemoveError(Exception): + def __init__(self, track_id: TrackId, message: str) -> None: + """Exception to be raised if track can not be removed. + + Args: + track_id (TrackId): the track id of the track to be removed. + message (str): the error message. + """ + super().__init__(message) + self._track_id = track_id + + +class RemoveMultipleTracksError(Exception): + """Exception to be raised if multiple tracks can not be removed. + + Args: + track_ids (list[TrackId]): the track id of the track to be removed. + message (str): the error message. + """ + + def __init__(self, track_ids: list[TrackId], message: str): + super().__init__(message) + self._track_ids = track_ids + + +class TrackRepository: + def __init__(self, dataset: TrackDataset) -> None: + self._dataset = dataset + self.observers = Subject[TrackRepositoryEvent]() + + def register_tracks_observer(self, observer: TrackListObserver) -> None: + """ + Listen to changes of the repository. + + Args: + observer (TrackListObserver): listener to be notified about changes + """ + self.observers.register(observer.notify_tracks) + + def add_all(self, tracks: Iterable[Track]) -> None: + """ + Add multiple tracks to the repository and notify only once about it. + + Args: + tracks (Iterable[Track]): tracks to be added + """ + self._dataset = self._dataset.add_all(tracks) + new_tracks = [track.id for track in tracks] + if new_tracks: + self.observers.notify(TrackRepositoryEvent(new_tracks, [])) + + def get_for(self, id: TrackId) -> Optional[Track]: + """ + Retrieve a track for the given id. + + Args: + id (TrackId): id to search for + + Returns: + Optional[Track]: track if it exists + """ + return self._dataset.get_for(id) + + def get_all(self) -> TrackDataset: + """ + Retrieve all tracks. + + Returns: + list[Track]: all tracks within the repository + """ + return self._dataset + + def get_all_ids(self) -> Iterable[TrackId]: + """Get all track ids in this repository. + + Returns: + Iterable[TrackId]: the track ids. + """ + return self._dataset.get_all_ids() + + def remove(self, track_id: TrackId) -> None: + """Remove track by its id and notify observers + + Raises: + TrackRemoveError: if track does not exist in repository. + + Args: + track_id (TrackId): the id of the track to be removed. + """ + try: + self._dataset = self._dataset.remove(track_id) + except KeyError: + raise TrackRemoveError( + track_id, f"Trying to remove non existing track with id '{track_id.id}'" + ) + # TODO: Pass removed track id to notify when moving observers to + # application layer + self.observers.notify(TrackRepositoryEvent([], [track_id])) + + def remove_multiple(self, track_ids: set[TrackId]) -> None: + failed_tracks: list[TrackId] = [] + for track_id in track_ids: + try: + self._dataset = self._dataset.remove(track_id) + except KeyError: + failed_tracks.append(track_id) + # TODO: Pass removed track id to notify when moving observers to + # application layer + + if failed_tracks: + raise RemoveMultipleTracksError( + failed_tracks, + ( + "Multiple tracks with following ids could not be removed." + f" '{[failed_track.id for failed_track in failed_tracks]}'" + ), + ) + self.observers.notify(TrackRepositoryEvent([], list(track_ids))) + + def split(self, chunks: int) -> Iterable[TrackDataset]: + return self._dataset.split(chunks) + + def clear(self) -> None: + """ + Clear the repository and inform the observers about the empty repository. + """ + removed = list(self._dataset.get_all_ids()) + self._dataset = self._dataset.clear() + self.observers.notify(TrackRepositoryEvent([], removed)) + + def __len__(self) -> int: + return len(self._dataset) + + +class TrackFileRepository: + def __init__(self) -> None: + self._files: set[Path] = set() + + def add(self, file: Path) -> None: + """ + Add a single track file the repository. + + Args: + file (Path): track file to be added. + """ + self._files.add(file) + + def add_all(self, files: Iterable[Path]) -> None: + """ + Add multiple files to the repository. + + Args: + files (Iterable[Path]): the files to be added. + """ + for file in files: + self.add(file) + + def get_all(self) -> set[Path]: + """ + Retrieve all track files. + + Returns: + set[Path]: all tracks within the repository. + """ + return self._files.copy() diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index fa8cdca07..e3a4b95d4 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -42,8 +42,8 @@ TrackHasNoDetectionError, TrackId, TrackImage, - TrackRepository, ) +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.video import PATH, SimpleVideo, Video, VideoReader from OTAnalytics.plugin_datastore.python_track_store import ( PythonDetection, diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index ac0e19831..50cbc3554 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -37,6 +37,8 @@ TrackId, TrackIdProvider, TrackImage, +) +from OTAnalytics.domain.track_repository import ( TrackListObserver, TrackRepository, TrackRepositoryEvent, diff --git a/OTAnalytics/plugin_ui/cli.py b/OTAnalytics/plugin_ui/cli.py index c52476394..cc3edac16 100644 --- a/OTAnalytics/plugin_ui/cli.py +++ b/OTAnalytics/plugin_ui/cli.py @@ -39,7 +39,7 @@ from OTAnalytics.domain.flow import Flow from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import Section, SectionType -from OTAnalytics.domain.track import TrackRepositoryEvent +from OTAnalytics.domain.track_repository import TrackRepositoryEvent from OTAnalytics.plugin_prototypes.eventlist_exporter.eventlist_exporter import ( AVAILABLE_EVENTLIST_EXPORTERS, OTC_CSV_FORMAT_NAME, diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py index ad980bb34..ce123fd63 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py @@ -78,7 +78,8 @@ SectionListObserver, SectionRepositoryEvent, ) -from OTAnalytics.domain.track import TrackImage, TrackListObserver, TrackRepositoryEvent +from OTAnalytics.domain.track import TrackImage +from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent from OTAnalytics.domain.types import EventType from OTAnalytics.domain.video import DifferentDrivesException, Video, VideoListObserver from OTAnalytics.plugin_ui.customtkinter_gui import toplevel_export_events diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 4bf80428b..6594b13d9 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -102,7 +102,7 @@ from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionRepository -from OTAnalytics.domain.track import TrackFileRepository, TrackRepository +from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository from OTAnalytics.plugin_datastore.python_track_store import ByMaxConfidence from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 330a574b9..29b49ffa1 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -30,7 +30,8 @@ from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import TrackIdProvider, TrackRepository +from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder from OTAnalytics.plugin_intersect.simple_intersect import ( SimpleTracksIntersectingSections, diff --git a/tests/OTAnalytics/application/analysis/test_traffic_counting.py b/tests/OTAnalytics/application/analysis/test_traffic_counting.py index 17d168a1d..2c2b51574 100644 --- a/tests/OTAnalytics/application/analysis/test_traffic_counting.py +++ b/tests/OTAnalytics/application/analysis/test_traffic_counting.py @@ -47,7 +47,8 @@ from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import Track, TrackId, TrackRepository +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.types import EventType from tests.conftest import TrackBuilder diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index 97e17d5bb..eefb28bd1 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -30,7 +30,8 @@ SectionId, SectionRepository, ) -from OTAnalytics.domain.track import TrackFileRepository, TrackImage, TrackRepository +from OTAnalytics.domain.track import TrackImage +from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.types import EventType from OTAnalytics.domain.video import SimpleVideo, Video, VideoReader, VideoRepository diff --git a/tests/OTAnalytics/application/test_state.py b/tests/OTAnalytics/application/test_state.py index 40ab5a341..c3abfa4f0 100644 --- a/tests/OTAnalytics/application/test_state.py +++ b/tests/OTAnalytics/application/test_state.py @@ -15,7 +15,6 @@ Plotter, SectionState, TrackImageUpdater, - TrackObserver, TracksMetadata, TrackState, TrackViewState, @@ -30,11 +29,9 @@ SectionRepositoryEvent, SectionType, ) -from OTAnalytics.domain.track import ( - Detection, - Track, - TrackId, - TrackImage, +from OTAnalytics.domain.track import Detection, Track, TrackId, TrackImage +from OTAnalytics.domain.track_repository import ( + TrackObserver, TrackRepository, TrackRepositoryEvent, ) diff --git a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py index 2fe725ace..d570e1c7d 100644 --- a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py +++ b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py @@ -30,13 +30,8 @@ from OTAnalytics.domain.filter import FilterElement from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import ( - Detection, - Track, - TrackId, - TrackIdProvider, - TrackRepository, -) +from OTAnalytics.domain.track import Detection, Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track_repository import TrackRepository @pytest.fixture diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 5d909bd9b..6a1cb553c 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -15,13 +15,8 @@ RemoveTracks, TrackRepositorySize, ) -from OTAnalytics.domain.track import ( - Track, - TrackDataset, - TrackFileRepository, - TrackId, - TrackRepository, -) +from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository @pytest.fixture diff --git a/tests/OTAnalytics/domain/test_track.py b/tests/OTAnalytics/domain/test_track.py index 6d7136dae..00d79cebc 100644 --- a/tests/OTAnalytics/domain/test_track.py +++ b/tests/OTAnalytics/domain/test_track.py @@ -3,11 +3,9 @@ import pytest -from OTAnalytics.domain.track import ( - Track, - TrackDataset, +from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track_repository import ( TrackFileRepository, - TrackId, TrackListObserver, TrackObserver, TrackRepository, diff --git a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py index 847e0eadf..10ef8830d 100644 --- a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py @@ -37,8 +37,8 @@ TrackClassificationCalculator, TrackId, TrackImage, - TrackRepository, ) +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.video import Video from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, diff --git a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py index d41738518..4f001ec16 100644 --- a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py @@ -2,7 +2,8 @@ import pytest -from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackRepository +from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, ) diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index e454ae7d8..605214efd 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -20,9 +20,8 @@ TrackId, TrackIdProvider, TrackImage, - TrackRepository, - TrackRepositoryEvent, ) +from OTAnalytics.domain.track_repository import TrackRepository, TrackRepositoryEvent from OTAnalytics.plugin_datastore.python_track_store import ( PythonDetection, PythonTrack, diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 29096558f..fae0dc002 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -60,7 +60,8 @@ from OTAnalytics.domain.event import EventRepository, SceneEventBuilder from OTAnalytics.domain.progress import NoProgressbarBuilder from OTAnalytics.domain.section import SectionId, SectionRepository, SectionType -from OTAnalytics.domain.track import TrackId, TrackRepository +from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonTrackDataset, diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index f7a2c9138..29e0beb79 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -20,7 +20,7 @@ from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.section import SectionRepository -from OTAnalytics.domain.track import TrackRepository +from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonTrackDataset, From 1461b13bb4f4c7e9fd06bc875f86c41c8e738e76 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 09:17:05 +0100 Subject: [PATCH 102/107] Move TrackDataset into separate module --- OTAnalytics/application/datastore.py | 3 +- OTAnalytics/application/eventlist.py | 3 +- .../use_cases/create_intersection_events.py | 2 +- .../application/use_cases/track_repository.py | 3 +- OTAnalytics/domain/intersect.py | 2 +- OTAnalytics/domain/track.py | 132 +---------------- OTAnalytics/domain/track_dataset.py | 136 ++++++++++++++++++ OTAnalytics/domain/track_repository.py | 3 +- .../plugin_datastore/python_track_store.py | 3 +- .../track_geometry_store/pygeos_store.py | 9 +- OTAnalytics/plugin_datastore/track_store.py | 3 +- .../multiprocessing.py | 2 +- .../sequential.py | 2 +- OTAnalytics/plugin_parser/otvision_parser.py | 2 +- OTAnalytics/plugin_parser/pandas_parser.py | 3 +- .../OTAnalytics/application/test_eventlist.py | 3 +- .../use_cases/test_create_events.py | 3 +- .../test_create_intersection_events.py | 3 +- .../use_cases/test_track_repository.py | 3 +- tests/OTAnalytics/domain/test_track.py | 3 +- .../plugin_datastore/test_track_store.py | 2 +- .../track_geometry_store/test_pygeos_store.py | 3 +- .../test_multiprocessing.py | 2 +- .../test_sequential.py | 2 +- tests/conftest.py | 3 +- 25 files changed, 173 insertions(+), 162 deletions(-) create mode 100644 OTAnalytics/domain/track_dataset.py diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index 98feef438..c9eefeaab 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -22,7 +22,8 @@ SectionListObserver, SectionRepository, ) -from OTAnalytics.domain.track import TrackDataset, TrackId, TrackImage +from OTAnalytics.domain.track import TrackId, TrackImage +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import ( TrackFileRepository, TrackListObserver, diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 106ea48a2..3667f0ab6 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -1,6 +1,7 @@ from OTAnalytics.domain.event import Event, EventType, SceneEventBuilder from OTAnalytics.domain.geometry import calculate_direction_vector -from OTAnalytics.domain.track import Detection, TrackDataset +from OTAnalytics.domain.track import Detection +from OTAnalytics.domain.track_dataset import TrackDataset class SceneActionDetector: diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 37d4dc2e2..564cbfb6c 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -14,7 +14,7 @@ ) from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy from OTAnalytics.domain.section import Area, LineSection, Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.types import EventType diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 2672f402d..46e4ffc8b 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -2,7 +2,8 @@ from typing import Iterable from OTAnalytics.application.logger import logger -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import ( RemoveMultipleTracksError, TrackFileRepository, diff --git a/OTAnalytics/domain/intersect.py b/OTAnalytics/domain/intersect.py index 37d59d169..41d9f47b3 100644 --- a/OTAnalytics/domain/intersect.py +++ b/OTAnalytics/domain/intersect.py @@ -4,7 +4,7 @@ from OTAnalytics.domain.event import Event, EventBuilder from OTAnalytics.domain.geometry import Coordinate, Line, Polygon from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset class IntersectImplementation(ABC): diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 713af6667..0c16caedf 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -1,13 +1,14 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable, Iterable, Iterator, Optional, Sequence +from typing import Callable, Iterable from PIL import Image from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset MIN_NUMBER_OF_DETECTIONS = 5 CLASSIFICATION: str = "classification" @@ -280,135 +281,6 @@ def calculate(self, detections: list[Detection]) -> str: raise NotImplementedError -@dataclass -class IntersectionPoint: - index: int - - -class TrackDataset(ABC): - def __iter__(self) -> Iterator[Track]: - yield from self.as_list() - - @abstractmethod - def add_all(self, other: Iterable[Track]) -> "TrackDataset": - raise NotImplementedError - - @abstractmethod - def get_all_ids(self) -> Iterable[TrackId]: - raise NotImplementedError - - @abstractmethod - def get_for(self, id: TrackId) -> Optional[Track]: - """ - Retrieve a track for the given id. - - Args: - id (TrackId): id to search for - - Returns: - Optional[Track]: track if it exists - """ - raise NotImplementedError - - @abstractmethod - def remove(self, track_id: TrackId) -> "TrackDataset": - raise NotImplementedError - - @abstractmethod - def clear(self) -> "TrackDataset": - """ - Return an empty version of the current TrackDataset. - """ - raise NotImplementedError - - @abstractmethod - def as_list(self) -> list[Track]: - raise NotImplementedError - - @abstractmethod - def intersecting_tracks( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: - """Return a set of tracks intersecting a set of sections. - - Args: - sections (list[Section]): the list of sections to intersect. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. - - Returns: - set[TrackId]: the track ids intersecting the given sections. - """ - raise NotImplementedError - - @abstractmethod - def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - """ - Return the intersection points resulting from the tracks and the - given sections. - - Args: - sections (list[Section]): the sections to intersect with. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. - - Returns: - dict[TrackId, list[tuple[SectionId]]]: the intersection points. - """ - raise NotImplementedError - - @abstractmethod - def contained_by_sections( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - """Return whether track coordinates are contained by the given sections. - - Args: - sections (list[Section]): the sections. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. - - Returns: - dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask of track - coordinates contained by given sections. - """ - raise NotImplementedError - - @abstractmethod - def split(self, chunks: int) -> Sequence["TrackDataset"]: - raise NotImplementedError - - @abstractmethod - def __len__(self) -> int: - """Number of tracks in the dataset.""" - raise NotImplementedError - - @abstractmethod - def filter_by_min_detection_length(self, length: int) -> "TrackDataset": - """Filter tracks by the minimum length of detections. - - Args: - length (int): minimum number detections a track should have. - - Returns: - TrackDataset: the filtered dataset. - """ - raise NotImplementedError - - @abstractmethod - def calculate_geometries_for( - self, offsets: Iterable[RelativeOffsetCoordinate] - ) -> None: - raise NotImplementedError - - @abstractmethod - def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: - raise NotImplementedError - - @abstractmethod - def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: - raise NotImplementedError - - class TrackIdProvider(ABC): """Interface to provide track ids.""" diff --git a/OTAnalytics/domain/track_dataset.py b/OTAnalytics/domain/track_dataset.py new file mode 100644 index 000000000..c3b1fbeca --- /dev/null +++ b/OTAnalytics/domain/track_dataset.py @@ -0,0 +1,136 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Iterator, Optional, Sequence + +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track import Track, TrackId + + +@dataclass +class IntersectionPoint: + index: int + + +class TrackDataset(ABC): + def __iter__(self) -> Iterator[Track]: + yield from self.as_list() + + @abstractmethod + def add_all(self, other: Iterable[Track]) -> "TrackDataset": + raise NotImplementedError + + @abstractmethod + def get_all_ids(self) -> Iterable[TrackId]: + raise NotImplementedError + + @abstractmethod + def get_for(self, id: TrackId) -> Optional[Track]: + """ + Retrieve a track for the given id. + + Args: + id (TrackId): id to search for + + Returns: + Optional[Track]: track if it exists + """ + raise NotImplementedError + + @abstractmethod + def remove(self, track_id: TrackId) -> "TrackDataset": + raise NotImplementedError + + @abstractmethod + def clear(self) -> "TrackDataset": + """ + Return an empty version of the current TrackDataset. + """ + raise NotImplementedError + + @abstractmethod + def as_list(self) -> list[Track]: + raise NotImplementedError + + @abstractmethod + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ + raise NotImplementedError + + @abstractmethod + def intersection_points( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """ + Return the intersection points resulting from the tracks and the + given sections. + + Args: + sections (list[Section]): the sections to intersect with. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. + + Returns: + dict[TrackId, list[tuple[SectionId]]]: the intersection points. + """ + raise NotImplementedError + + @abstractmethod + def contained_by_sections( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: + """Return whether track coordinates are contained by the given sections. + + Args: + sections (list[Section]): the sections. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. + + Returns: + dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask of track + coordinates contained by given sections. + """ + raise NotImplementedError + + @abstractmethod + def split(self, chunks: int) -> Sequence["TrackDataset"]: + raise NotImplementedError + + @abstractmethod + def __len__(self) -> int: + """Number of tracks in the dataset.""" + raise NotImplementedError + + @abstractmethod + def filter_by_min_detection_length(self, length: int) -> "TrackDataset": + """Filter tracks by the minimum length of detections. + + Args: + length (int): minimum number detections a track should have. + + Returns: + TrackDataset: the filtered dataset. + """ + raise NotImplementedError + + @abstractmethod + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + raise NotImplementedError + + @abstractmethod + def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: + raise NotImplementedError + + @abstractmethod + def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: + raise NotImplementedError diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py index 5af83973f..0e9b74a59 100644 --- a/OTAnalytics/domain/track_repository.py +++ b/OTAnalytics/domain/track_repository.py @@ -4,7 +4,8 @@ from typing import Iterable, Optional from OTAnalytics.domain.observer import Subject -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset @dataclass(frozen=True) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 6bfc42155..ab19d1aab 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -17,14 +17,13 @@ from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, Detection, - IntersectionPoint, Track, TrackClassificationCalculator, - TrackDataset, TrackGeometryDataset, TrackHasNoDetectionError, TrackId, ) +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 48dde1c4f..0435316e1 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -22,13 +22,8 @@ from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import ( - IntersectionPoint, - Track, - TrackDataset, - TrackGeometryDataset, - TrackId, -) +from OTAnalytics.domain.track import Track, TrackGeometryDataset, TrackId +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset TRACK_ID = "track_id" diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index aae23a742..545ea3e74 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -25,12 +25,11 @@ from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, Detection, - IntersectionPoint, Track, - TrackDataset, TrackGeometryDataset, TrackId, ) +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType diff --git a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py index faf9cad99..b1cf4a152 100644 --- a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py +++ b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py @@ -7,7 +7,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset class MultiprocessingIntersectParallelization(IntersectParallelizationStrategy): diff --git a/OTAnalytics/plugin_intersect_parallelization/sequential.py b/OTAnalytics/plugin_intersect_parallelization/sequential.py index 6c8d9aa9b..7675fc63a 100644 --- a/OTAnalytics/plugin_intersect_parallelization/sequential.py +++ b/OTAnalytics/plugin_intersect_parallelization/sequential.py @@ -3,7 +3,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset class SequentialIntersect(IntersectParallelizationStrategy): diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index e3a4b95d4..5643623c3 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -38,11 +38,11 @@ Detection, Track, TrackClassificationCalculator, - TrackDataset, TrackHasNoDetectionError, TrackId, TrackImage, ) +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.video import PATH, SimpleVideo, Video, VideoReader from OTAnalytics.plugin_datastore.python_track_store import ( diff --git a/OTAnalytics/plugin_parser/pandas_parser.py b/OTAnalytics/plugin_parser/pandas_parser.py index de6384cf6..a1f2c4cce 100644 --- a/OTAnalytics/plugin_parser/pandas_parser.py +++ b/OTAnalytics/plugin_parser/pandas_parser.py @@ -5,7 +5,8 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain import track -from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackDataset, TrackId +from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasTrackClassificationCalculator, PandasTrackDataset, diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index 751e1a792..bcd437bb3 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -12,7 +12,8 @@ RelativeOffsetCoordinate, ) from OTAnalytics.domain.section import LineSection, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack diff --git a/tests/OTAnalytics/application/use_cases/test_create_events.py b/tests/OTAnalytics/application/use_cases/test_create_events.py index 3da21d66b..ba32aeb8b 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_events.py @@ -16,7 +16,8 @@ from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.event import Event from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Track, TrackDataset +from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track_dataset import TrackDataset @pytest.fixture diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 52bf45f1e..89445b985 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -21,7 +21,8 @@ SectionId, SectionType, ) -from OTAnalytics.domain.track import Detection, IntersectionPoint, Track, TrackDataset +from OTAnalytics.domain.track import Detection, Track +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType from tests.conftest import TrackBuilder diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 6a1cb553c..b6561d4e0 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -15,7 +15,8 @@ RemoveTracks, TrackRepositorySize, ) -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository diff --git a/tests/OTAnalytics/domain/test_track.py b/tests/OTAnalytics/domain/test_track.py index 00d79cebc..4a2a3daaa 100644 --- a/tests/OTAnalytics/domain/test_track.py +++ b/tests/OTAnalytics/domain/test_track.py @@ -3,7 +3,8 @@ import pytest -from OTAnalytics.domain.track import Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import ( TrackFileRepository, TrackListObserver, diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 29892354b..69499f86c 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -14,10 +14,10 @@ from OTAnalytics.domain.track import ( TRACK_GEOMETRY_FACTORY, Track, - TrackDataset, TrackGeometryDataset, TrackId, ) +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 0a42f48da..23f9988ef 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -15,14 +15,13 @@ from OTAnalytics.domain.track import ( TRACK_CLASSIFICATION, TRACK_GEOMETRY_FACTORY, - IntersectionPoint, Track, - TrackDataset, TrackGeometryDataset, TrackId, X, Y, ) +from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( BASE_GEOMETRY, diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py index 8cf8a2df4..60eb99821 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py @@ -5,7 +5,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.multiprocessing import ( MultiprocessingIntersectParallelization, ) diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py index 500b9d708..b4bfa3137 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py @@ -3,7 +3,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.section import Section -from OTAnalytics.domain.track import TrackDataset +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.sequential import SequentialIntersect diff --git a/tests/conftest.py b/tests/conftest.py index b665bac18..6735d1034 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,8 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackDataset, TrackId +from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonDetection, PythonTrack from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( From 8f56e69d06b17daeab199d0d1b1f5a4a9e6aa3fe Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 09:20:29 +0100 Subject: [PATCH 103/107] Move TrackGeometryDataset into track_dataset module --- OTAnalytics/domain/track.py | 130 +----------------- OTAnalytics/domain/track_dataset.py | 126 +++++++++++++++++ .../plugin_datastore/python_track_store.py | 9 +- .../track_geometry_store/pygeos_store.py | 8 +- OTAnalytics/plugin_datastore/track_store.py | 9 +- OTAnalytics/plugin_parser/pandas_parser.py | 4 +- .../OTAnalytics/plugin_datastore/conftest.py | 6 +- .../test_python_track_storage.py | 9 +- .../plugin_datastore/test_track_store.py | 7 +- .../track_geometry_store/test_pygeos_store.py | 11 +- .../plugin_parser/test_pandas_parser.py | 2 +- 11 files changed, 160 insertions(+), 161 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 0c16caedf..1bced9fc1 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -1,14 +1,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime -from typing import Callable, Iterable +from typing import Iterable from PIL import Image from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset MIN_NUMBER_OF_DETECTIONS = 5 CLASSIFICATION: str = "classification" @@ -340,129 +338,3 @@ def reset(self) -> None: All configurations made to the builder will be reset. """ raise NotImplementedError - - -class TrackGeometryDataset(ABC): - """Dataset containing track geometries. - - Only tracks of size greater equal two are contained in the dataset. - Tracks of size less than two will not be contained in the dataset - since it is not possible to construct a track with less than two - coordinates. - """ - - @property - @abstractmethod - def track_ids(self) -> set[str]: - """Get track ids of tracks stored in dataset. - - Returns: - set[str]: the track ids stored. - """ - raise NotImplementedError - - @property - @abstractmethod - def offset(self) -> RelativeOffsetCoordinate: - raise NotImplementedError - - @property - @abstractmethod - def empty(self) -> bool: - raise NotImplementedError - - @staticmethod - @abstractmethod - def from_track_dataset( - dataset: TrackDataset, offset: RelativeOffsetCoordinate - ) -> "TrackGeometryDataset": - raise NotImplementedError - - @abstractmethod - def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": - """Add tracks to existing dataset. - - Pre-existing tracks will be overwritten. - - Args: - tracks (Iterable[Track]): the tracks to add. - - Returns: - TrackGeometryDataset: the dataset with tracks added. - - """ - raise NotImplementedError - - @abstractmethod - def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": - """Remove track geometries with given ids from dataset. - - Args: - ids (Iterable[TrackId]): the track geometries to remove. - - Returns: - TrackGeometryDataset: the dataset with tracks removed. - """ - raise NotImplementedError - - @abstractmethod - def get_for(self, track_ids: Iterable[str]) -> "TrackGeometryDataset": - """Get geometries for given track ids if they exist. - - Ids that do not exist will not be included in the dataset. - - Args: - track_ids (Iterable[str]): the track ids. - - Returns: - TrackGeometryDataset: the dataset with tracks. - """ - raise NotImplementedError - - @abstractmethod - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - """Return a set of tracks intersecting a set of sections. - - Args: - sections (list[Section]): the list of sections to intersect. - - Returns: - set[TrackId]: the track ids intersecting the given sections. - """ - raise NotImplementedError - - @abstractmethod - def intersection_points( - self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - """ - Return the intersection points resulting from the tracks and the - given sections. - - Args: - sections (list[Section]): the sections to intersect with. - - Returns: - dict[TrackId, list[tuple[SectionId]]]: the intersection points. - """ - raise NotImplementedError - - @abstractmethod - def contained_by_sections( - self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - """Return whether track coordinates are contained by the given sections. - - Args: - sections (list[Section]): the sections. - - Returns: - dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask - of track coordinates contained by given sections. - """ - raise NotImplementedError - - -TRACK_GEOMETRY_FACTORY = Callable[ - [TrackDataset, RelativeOffsetCoordinate], TrackGeometryDataset -] diff --git a/OTAnalytics/domain/track_dataset.py b/OTAnalytics/domain/track_dataset.py index c3b1fbeca..396121871 100644 --- a/OTAnalytics/domain/track_dataset.py +++ b/OTAnalytics/domain/track_dataset.py @@ -134,3 +134,129 @@ def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: @abstractmethod def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: raise NotImplementedError + + +class TrackGeometryDataset(ABC): + """Dataset containing track geometries. + + Only tracks of size greater equal two are contained in the dataset. + Tracks of size less than two will not be contained in the dataset + since it is not possible to construct a track with less than two + coordinates. + """ + + @property + @abstractmethod + def track_ids(self) -> set[str]: + """Get track ids of tracks stored in dataset. + + Returns: + set[str]: the track ids stored. + """ + raise NotImplementedError + + @property + @abstractmethod + def offset(self) -> RelativeOffsetCoordinate: + raise NotImplementedError + + @property + @abstractmethod + def empty(self) -> bool: + raise NotImplementedError + + @staticmethod + @abstractmethod + def from_track_dataset( + dataset: TrackDataset, offset: RelativeOffsetCoordinate + ) -> "TrackGeometryDataset": + raise NotImplementedError + + @abstractmethod + def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": + """Add tracks to existing dataset. + + Pre-existing tracks will be overwritten. + + Args: + tracks (Iterable[Track]): the tracks to add. + + Returns: + TrackGeometryDataset: the dataset with tracks added. + + """ + raise NotImplementedError + + @abstractmethod + def remove(self, ids: Iterable[TrackId]) -> "TrackGeometryDataset": + """Remove track geometries with given ids from dataset. + + Args: + ids (Iterable[TrackId]): the track geometries to remove. + + Returns: + TrackGeometryDataset: the dataset with tracks removed. + """ + raise NotImplementedError + + @abstractmethod + def get_for(self, track_ids: Iterable[str]) -> "TrackGeometryDataset": + """Get geometries for given track ids if they exist. + + Ids that do not exist will not be included in the dataset. + + Args: + track_ids (Iterable[str]): the track ids. + + Returns: + TrackGeometryDataset: the dataset with tracks. + """ + raise NotImplementedError + + @abstractmethod + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ + raise NotImplementedError + + @abstractmethod + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """ + Return the intersection points resulting from the tracks and the + given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId]]]: the intersection points. + """ + raise NotImplementedError + + @abstractmethod + def contained_by_sections( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: + """Return whether track coordinates are contained by the given sections. + + Args: + sections (list[Section]): the sections. + + Returns: + dict[TrackId, list[tuple[SectionId, list[bool]]]]: boolean mask + of track coordinates contained by given sections. + """ + raise NotImplementedError + + +TRACK_GEOMETRY_FACTORY = Callable[ + [TrackDataset, RelativeOffsetCoordinate], TrackGeometryDataset +] diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index ab19d1aab..be89f1243 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -15,15 +15,18 @@ ) from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import ( - TRACK_GEOMETRY_FACTORY, Detection, Track, TrackClassificationCalculator, - TrackGeometryDataset, TrackHasNoDetectionError, TrackId, ) -from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset +from OTAnalytics.domain.track_dataset import ( + TRACK_GEOMETRY_FACTORY, + IntersectionPoint, + TrackDataset, + TrackGeometryDataset, +) from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py index 0435316e1..02f5c0447 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pygeos_store.py @@ -22,8 +22,12 @@ from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate, apply_offset from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Track, TrackGeometryDataset, TrackId -from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import ( + IntersectionPoint, + TrackDataset, + TrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset TRACK_ID = "track_id" diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 545ea3e74..2e6c10286 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -22,14 +22,13 @@ calculate_direction_vector, ) from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import ( +from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track_dataset import ( TRACK_GEOMETRY_FACTORY, - Detection, - Track, + IntersectionPoint, + TrackDataset, TrackGeometryDataset, - TrackId, ) -from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType diff --git a/OTAnalytics/plugin_parser/pandas_parser.py b/OTAnalytics/plugin_parser/pandas_parser.py index a1f2c4cce..6b8fa7f55 100644 --- a/OTAnalytics/plugin_parser/pandas_parser.py +++ b/OTAnalytics/plugin_parser/pandas_parser.py @@ -5,8 +5,8 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain import track -from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, TrackId -from OTAnalytics.domain.track_dataset import TrackDataset +from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset import TRACK_GEOMETRY_FACTORY, TrackDataset from OTAnalytics.plugin_datastore.track_store import ( PandasTrackClassificationCalculator, PandasTrackDataset, diff --git a/tests/OTAnalytics/plugin_datastore/conftest.py b/tests/OTAnalytics/plugin_datastore/conftest.py index eb8f91f41..223531768 100644 --- a/tests/OTAnalytics/plugin_datastore/conftest.py +++ b/tests/OTAnalytics/plugin_datastore/conftest.py @@ -3,7 +3,11 @@ import pytest -from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY, Track, TrackGeometryDataset +from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track_dataset import ( + TRACK_GEOMETRY_FACTORY, + TrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, ) diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py index d867a730a..e4a93f017 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_storage.py @@ -11,13 +11,8 @@ RelativeOffsetCoordinate, calculate_direction_vector, ) -from OTAnalytics.domain.track import ( - Detection, - Track, - TrackGeometryDataset, - TrackHasNoDetectionError, - TrackId, -) +from OTAnalytics.domain.track import Detection, Track, TrackHasNoDetectionError, TrackId +from OTAnalytics.domain.track_dataset import TrackGeometryDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 69499f86c..0a6bdc2ea 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -11,13 +11,12 @@ RelativeOffsetCoordinate, calculate_direction_vector, ) -from OTAnalytics.domain.track import ( +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset import ( TRACK_GEOMETRY_FACTORY, - Track, + TrackDataset, TrackGeometryDataset, - TrackId, ) -from OTAnalytics.domain.track_dataset import TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py index 23f9988ef..cb1e74b43 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pygeos_store.py @@ -12,16 +12,13 @@ from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Area, LineSection, Section, SectionId -from OTAnalytics.domain.track import ( - TRACK_CLASSIFICATION, +from OTAnalytics.domain.track import TRACK_CLASSIFICATION, Track, TrackId, X, Y +from OTAnalytics.domain.track_dataset import ( TRACK_GEOMETRY_FACTORY, - Track, + IntersectionPoint, + TrackDataset, TrackGeometryDataset, - TrackId, - X, - Y, ) -from OTAnalytics.domain.track_dataset import IntersectionPoint, TrackDataset from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( BASE_GEOMETRY, diff --git a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py index 4f001ec16..ce4be43ee 100644 --- a/tests/OTAnalytics/plugin_parser/test_pandas_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_pandas_parser.py @@ -2,7 +2,7 @@ import pytest -from OTAnalytics.domain.track import TRACK_GEOMETRY_FACTORY +from OTAnalytics.domain.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.track_geometry_store.pygeos_store import ( PygeosTrackGeometryDataset, From 7c69210f6314188f0a7473d1ffb950ddb5f571a6 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 09:32:11 +0100 Subject: [PATCH 104/107] Use correct type hint --- OTAnalytics/domain/track_dataset.py | 7 ++++--- OTAnalytics/plugin_datastore/python_track_store.py | 6 +++--- OTAnalytics/plugin_datastore/track_store.py | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/domain/track_dataset.py b/OTAnalytics/domain/track_dataset.py index 396121871..56d2d86f1 100644 --- a/OTAnalytics/domain/track_dataset.py +++ b/OTAnalytics/domain/track_dataset.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, Callable, Iterable, Iterator, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence +from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track, TrackId @@ -128,11 +129,11 @@ def calculate_geometries_for( raise NotImplementedError @abstractmethod - def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_first_segments(self, consumer: Callable[[Event], None]) -> None: raise NotImplementedError @abstractmethod - def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_last_segments(self, consumer: Callable[[Event], None]) -> None: raise NotImplementedError diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index be89f1243..66b3b0ec8 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Any, Callable, Iterable, Optional, Sequence +from typing import Callable, Iterable, Optional, Sequence from more_itertools import batched @@ -424,7 +424,7 @@ def calculate_geometries_for( if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) - def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_first_segments(self, consumer: Callable[[Event], None]) -> None: for track in self.as_list(): event = self.__create_enter_scene_event(track) consumer(event) @@ -450,7 +450,7 @@ def __create_enter_scene_event(self, track: Track) -> Event: video_name=track.first_detection.video_name, ) - def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_last_segments(self, consumer: Callable[[Event], None]) -> None: for track in self.as_list(): event = self.__create_leave_scene_event(track) consumer(event) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 2e6c10286..a1b80c459 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -408,7 +408,7 @@ def calculate_geometries_for( if offset not in self._geometry_datasets.keys(): self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) - def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_first_segments(self, consumer: Callable[[Event], None]) -> None: first_detections = self._dataset.groupby(level=0, group_keys=True) self._dataset["next_x"] = first_detections[track.X].shift(-1) self._dataset["next_y"] = first_detections[track.Y].shift(-1) @@ -438,7 +438,7 @@ def apply_to_first_segments(self, consumer: Callable[[Any], None]) -> None: ) consumer(event) - def apply_to_last_segments(self, consumer: Callable[[Any], None]) -> None: + def apply_to_last_segments(self, consumer: Callable[[Event], None]) -> None: first_detections = self._dataset.groupby(level=0, group_keys=True) self._dataset["previous_x"] = first_detections[track.X].shift(1) self._dataset["previous_y"] = first_detections[track.Y].shift(1) From d4392b8739cea7d810f4ffd751b792a8c6ed0b31 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 9 Jan 2024 17:34:51 +0100 Subject: [PATCH 105/107] Remove unused code --- OTAnalytics/application/eventlist.py | 57 +----------------- OTAnalytics/plugin_ui/main_application.py | 4 +- .../OTAnalytics/application/test_eventlist.py | 60 +------------------ tests/OTAnalytics/plugin_ui/test_cli.py | 4 +- 4 files changed, 9 insertions(+), 116 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 3667f0ab6..0229fbd45 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -1,62 +1,9 @@ -from OTAnalytics.domain.event import Event, EventType, SceneEventBuilder -from OTAnalytics.domain.geometry import calculate_direction_vector -from OTAnalytics.domain.track import Detection +from OTAnalytics.domain.event import Event from OTAnalytics.domain.track_dataset import TrackDataset class SceneActionDetector: - """Detect when a road user enters or leaves the scene. - - Args: - scene_event_builder (SceneEventBuilder): the builder to build scene events - """ - - def __init__(self, scene_event_builder: SceneEventBuilder) -> None: - self._event_builder = scene_event_builder - - def detect_enter_scene( - self, from_detection: Detection, to_detection: Detection, classification: str - ) -> Event: - """Detect the first time a road user enters the scene. - - Args: - tracks (Track): the track associated with the road user - - Returns: - Iterable[Event]: the enter scene event - """ - self._event_builder.add_event_type(EventType.ENTER_SCENE) - self._event_builder.add_road_user_type(classification) - self._event_builder.add_direction_vector( - calculate_direction_vector( - from_detection.x, from_detection.y, to_detection.x, to_detection.y - ) - ) - self._event_builder.add_event_coordinate(from_detection.x, from_detection.y) - - return self._event_builder.create_event(from_detection) - - def detect_leave_scene( - self, from_detection: Detection, to_detection: Detection, classification: str - ) -> Event: - """Detect the last time a road user is seen before leaving the scene. - - Args: - tracks (Track): the track associated with the road user - - Returns: - Iterable[Event]: the leave scene event - """ - self._event_builder.add_event_type(EventType.LEAVE_SCENE) - self._event_builder.add_road_user_type(classification) - self._event_builder.add_direction_vector( - calculate_direction_vector( - from_detection.x, from_detection.y, to_detection.x, to_detection.y - ) - ) - self._event_builder.add_event_coordinate(to_detection.x, to_detection.y) - - return self._event_builder.create_event(to_detection) + """Detect when a road user enters or leaves the scene.""" def detect(self, tracks: TrackDataset) -> list[Event]: """Detect all enter and leave scene events. diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 6594b13d9..1a694ef85 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -97,7 +97,7 @@ ) from OTAnalytics.application.use_cases.update_project import ProjectUpdater from OTAnalytics.application.use_cases.video_repository import ClearAllVideos -from OTAnalytics.domain.event import EventRepository, SceneEventBuilder +from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.filter import FilterElementSettingRestorer from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder @@ -720,7 +720,7 @@ def _create_use_case_create_events( create_intersection_events = SimpleCreateIntersectionEvents( run_intersect, section_provider, add_events ) - scene_action_detector = SceneActionDetector(SceneEventBuilder()) + scene_action_detector = SceneActionDetector() create_scene_events = SimpleCreateSceneEvents( get_all_tracks_without_single_detections, scene_action_detector, add_events ) diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index bcd437bb3..623d3f5f8 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -4,13 +4,8 @@ import pytest from OTAnalytics.application.eventlist import SceneActionDetector -from OTAnalytics.domain.event import Event, EventType, SceneEventBuilder -from OTAnalytics.domain.geometry import ( - Coordinate, - DirectionVector2D, - ImageCoordinate, - RelativeOffsetCoordinate, -) +from OTAnalytics.domain.event import EventType +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import LineSection, SectionId from OTAnalytics.domain.track import Detection, Track, TrackId from OTAnalytics.domain.track_dataset import TrackDataset @@ -121,63 +116,14 @@ def line_section() -> LineSection: class TestSceneActionDetector: - def test_detect_enter_scene(self, track_1: Track) -> None: - from_detection = track_1.detections[0] - to_detection = track_1.detections[1] - classification = track_1.classification - scene_event_builder = SceneEventBuilder() - scene_event_builder.add_event_type(EventType.ENTER_SCENE) - scene_event_builder.add_road_user_type("car") - scene_action_detector = SceneActionDetector(scene_event_builder) - event = scene_action_detector.detect_enter_scene( - from_detection, to_detection, classification - ) - assert event == Event( - road_user_id="1", - road_user_type="car", - hostname="myhostname", - occurrence=datetime(2022, 1, 1, 0, 0, 0, 0), - frame_number=1, - section_id=None, - event_coordinate=ImageCoordinate(0.0, 5.0), - event_type=EventType.ENTER_SCENE, - direction_vector=DirectionVector2D(10, 0), - video_name="myhostname_something.mp4", - ) - - def test_detect_leave_scene(self, track_1: Track) -> None: - from_detection = track_1.detections[-2] - to_detection = track_1.detections[-1] - classification = track_1.classification - scene_event_builder = SceneEventBuilder() - scene_event_builder.add_event_type(EventType.LEAVE_SCENE) - scene_event_builder.add_road_user_type("car") - scene_action_detector = SceneActionDetector(scene_event_builder) - event = scene_action_detector.detect_leave_scene( - from_detection, to_detection, classification - ) - assert event == Event( - road_user_id="1", - road_user_type="car", - hostname="myhostname", - occurrence=datetime(2022, 1, 1, 0, 0, 0, 4), - frame_number=5, - section_id=None, - event_coordinate=ImageCoordinate(25, 5), - event_type=EventType.LEAVE_SCENE, - direction_vector=DirectionVector2D(5, 0), - video_name="myhostname_something.mp4", - ) - def test_detect( self, track_1: Track, track_2: Track, ) -> None: mock_tracks = Mock(spec=TrackDataset) - mock_event_builder = Mock(spec=SceneEventBuilder) - scene_action_detector = SceneActionDetector(mock_event_builder) + scene_action_detector = SceneActionDetector() scene_action_detector.detect(mock_tracks) mock_tracks.apply_to_first_segments.assert_called_once() diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index fae0dc002..e1c07e7ee 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -57,7 +57,7 @@ GetTracksWithoutSingleDetections, RemoveTracks, ) -from OTAnalytics.domain.event import EventRepository, SceneEventBuilder +from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.progress import NoProgressbarBuilder from OTAnalytics.domain.section import SectionId, SectionRepository, SectionType from OTAnalytics.domain.track import TrackId @@ -311,7 +311,7 @@ def cli_dependencies(self) -> dict[str, Any]: ) create_scene_events = SimpleCreateSceneEvents( get_tracks_without_single_detections, - SceneActionDetector(SceneEventBuilder()), + SceneActionDetector(), add_events, ) create_events = CreateEvents( From 90eaaa5b43fcc361d82bcf2ea3701a4c520db2f7 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:46:57 +0100 Subject: [PATCH 106/107] Rename test file --- .../domain/{test_track.py => test_track_repository.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/OTAnalytics/domain/{test_track.py => test_track_repository.py} (100%) diff --git a/tests/OTAnalytics/domain/test_track.py b/tests/OTAnalytics/domain/test_track_repository.py similarity index 100% rename from tests/OTAnalytics/domain/test_track.py rename to tests/OTAnalytics/domain/test_track_repository.py From c3da16b453afe8dca167d67b1214031291ee8730 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:49:59 +0100 Subject: [PATCH 107/107] Remove resolved todo comment --- OTAnalytics/plugin_prototypes/track_visualization/track_viz.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 50cbc3554..5b45b5dc7 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -287,9 +287,6 @@ def get_data(self) -> DataFrame: ) ids = [track_id.id for track_id in self._filter.get_ids()] - # TODO: This only works for DataFrames with track id and occurrence as - # an multi-index. Could not be working with a CachedPandasTrackProvider - # since no such multi-index is being used there. intersection_of_ids = data.index.get_level_values(0).unique().intersection(ids) return data.loc[intersection_of_ids]