From ecc7f4bde61ed81d32dab04019ccf2d160769f86 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 09:19:43 +0200 Subject: [PATCH 01/49] Show bounding boxes for current frame --- OTAnalytics/adapter_ui/view_model.py | 8 ++ OTAnalytics/application/application.py | 8 ++ OTAnalytics/application/datastore.py | 15 ++- OTAnalytics/application/plotting.py | 4 +- OTAnalytics/application/state.py | 11 +++ OTAnalytics/domain/track.py | 3 + OTAnalytics/domain/video.py | 18 ++++ OTAnalytics/plugin_parser/otvision_parser.py | 20 ++-- .../track_visualization/track_viz.py | 67 +++++++++++--- .../customtkinter_gui/dummy_viewmodel.py | 6 ++ OTAnalytics/plugin_ui/main_application.py | 92 +++++++++++++++---- .../plugin_video_processing/video_reader.py | 12 ++- .../test_video_reader.py | 11 +-- 13 files changed, 224 insertions(+), 51 deletions(-) diff --git a/OTAnalytics/adapter_ui/view_model.py b/OTAnalytics/adapter_ui/view_model.py index 16be0d692..f69448811 100644 --- a/OTAnalytics/adapter_ui/view_model.py +++ b/OTAnalytics/adapter_ui/view_model.py @@ -307,3 +307,11 @@ def switch_to_next_date_range(self) -> None: @abstractmethod def export_counts(self) -> None: raise NotImplementedError + + @abstractmethod + def next_frame(self) -> None: + pass + + @abstractmethod + def previous_frame(self) -> None: + pass diff --git a/OTAnalytics/application/application.py b/OTAnalytics/application/application.py index 1b35ab046..94dd92b7b 100644 --- a/OTAnalytics/application/application.py +++ b/OTAnalytics/application/application.py @@ -524,6 +524,14 @@ def get_current_track_offset(self) -> Optional[RelativeOffsetCoordinate]: """ return self.track_view_state.track_offset.get() + def next_frame(self) -> None: + if current := self.track_view_state.frame.get(): + self.track_view_state.frame.set(current + 1) + + def previous_frame(self) -> None: + if current := self.track_view_state.frame.get(): + self.track_view_state.frame.set(current - 1) + def update_date_range_tracks_filter(self, date_range: DateRange) -> None: """Update the date range of the track filter. diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index 9cc484e0c..de5a8d2dc 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Iterable, Optional, Sequence, Tuple @@ -92,7 +93,7 @@ def serialize( class VideoParser(ABC): @abstractmethod - def parse(self, file: Path) -> Video: + def parse(self, file: Path, start_date: Optional[datetime]) -> Video: pass @abstractmethod @@ -296,7 +297,7 @@ def load_video_files(self, files: list[Path]) -> None: videos = [] for file in files: try: - videos.append(self._video_parser.parse(file)) + videos.append(self._video_parser.parse(file, None)) except Exception as cause: raised_exceptions.append(cause) if raised_exceptions: @@ -556,7 +557,11 @@ def get_video_for(self, track_id: TrackId) -> Optional[Video]: def get_all_videos(self) -> list[Video]: return self._video_repository.get_all() - def get_image_of_track(self, track_id: TrackId) -> Optional[TrackImage]: + def get_image_of_track( + self, + track_id: TrackId, + frame: int = 0, + ) -> Optional[TrackImage]: """ Retrieve an image for the given track. @@ -567,4 +572,6 @@ def get_image_of_track(self, track_id: TrackId) -> Optional[TrackImage]: Optional[TrackImage]: an image of the track if the track is available and the image can be loaded """ - return video.get_frame(0) if (video := self.get_video_for(track_id)) else None + return ( + video.get_frame(frame) if (video := self.get_video_for(track_id)) else None + ) diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index 45179a735..c2386d7b3 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -92,7 +92,9 @@ def __init__(self, track_view_state: TrackViewState, datastore: Datastore) -> No self._datastore = datastore def plot(self) -> Optional[TrackImage]: + current_frame = self._track_view_state.frame.get() + frame_number = current_frame if current_frame else 1 if videos := self._track_view_state.selected_videos.get(): if len(videos) > 0: - return videos[0].get_frame(0) + return videos[0].get_frame(frame_number) return None diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index cb4870680..8fa5a6a9d 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -184,6 +184,7 @@ def __init__(self) -> None: self.selected_videos: ObservableProperty[list[Video]] = ObservableProperty[ list[Video] ](default=[]) + self.frame = ObservableProperty[int](1) class TrackPropertiesUpdater: @@ -301,6 +302,7 @@ def __init__( self._plotter = plotter self._track_view_state.show_tracks.register(self._notify_show_tracks) self._track_view_state.track_offset.register(self._notify_track_offset) + self._track_view_state.frame.register(self._notify_frame) self._track_view_state.filter_element.register(self._notify_filter_element) self._section_state.selected_sections.register(self._notify_section_selection) self._flow_state.selected_flows.register(self._notify_flow_changed) @@ -341,6 +343,15 @@ def _notify_track_offset(self, offset: Optional[RelativeOffsetCoordinate]) -> No """ self._update() + def _notify_frame(self, frame: Optional[int]) -> None: + """ + Will update the image according to changes of the track offset property. + + Args: + offset (Optional[RelativeOffsetCoordinate]): current value + """ + self._update() + def _notify_filter_element(self, _: FilterElement) -> None: """ Will update the image according to changes of the filter element. diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 0426a7d1f..f18221101 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -273,6 +273,9 @@ def height(self) -> int: """ pass + def save(self, name: str) -> None: + self.as_image().save(name) + @dataclass(frozen=True) class PilImage(TrackImage): diff --git a/OTAnalytics/domain/video.py b/OTAnalytics/domain/video.py index 868a8e1e4..11f7f2b2d 100644 --- a/OTAnalytics/domain/video.py +++ b/OTAnalytics/domain/video.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from datetime import datetime, timedelta from os import path from os.path import normcase, splitdrive from pathlib import Path @@ -23,6 +24,10 @@ def get_frame(self, video: Path, index: int) -> TrackImage: """ pass + @abstractmethod + def get_frame_number_for(self, video_path: Path, date: timedelta) -> int: + raise NotImplementedError + class Video(ABC): @abstractmethod @@ -33,6 +38,10 @@ def get_path(self) -> Path: def get_frame(self, index: int) -> TrackImage: pass + @abstractmethod + def get_frame_number_for(self, date: datetime) -> int: + raise NotImplementedError + @abstractmethod def to_dict( self, @@ -73,6 +82,7 @@ class SimpleVideo(Video): video_reader: VideoReader path: Path + start_date: Optional[datetime] def __post_init__(self) -> None: self.check_path_exists() @@ -95,6 +105,14 @@ def get_frame(self, index: int) -> TrackImage: """ return self.video_reader.get_frame(self.path, index) + def get_frame_number_for(self, date: datetime) -> int: + if not self.start_date: + return 0 + time_in_video = date - self.start_date + if time_in_video < timedelta(0): + return 0 + return self.video_reader.get_frame_number_for(self.path, time_in_video) + def to_dict( self, relative_to: Path, diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index 636a0f6a4..c46258b2c 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -669,8 +669,8 @@ class SimpleVideoParser(VideoParser): def __init__(self, video_reader: VideoReader) -> None: self._video_reader = video_reader - def parse(self, file: Path) -> Video: - return SimpleVideo(self._video_reader, file) + def parse(self, file: Path, start_date: Optional[datetime]) -> Video: + return SimpleVideo(self._video_reader, file, start_date) def parse_list( self, @@ -687,7 +687,7 @@ def __create_video( if PATH not in entry: raise MissingPath(entry) video_path = Path(base_folder, entry[PATH]) - return self.parse(video_path) + return self.parse(video_path, None) def convert( self, @@ -714,6 +714,9 @@ def get_frame(self, index: int) -> TrackImage: self.cache[index] = new_frame return new_frame + def get_frame_number_for(self, date: datetime) -> int: + return self.other.get_frame_number_for(date) + def to_dict(self, relative_to: Path) -> dict: return self.other.to_dict(relative_to) @@ -722,8 +725,8 @@ class CachedVideoParser(VideoParser): def __init__(self, other: VideoParser) -> None: self._other = other - def parse(self, file: Path) -> Video: - other_video = self._other.parse(file) + def parse(self, file: Path, start_date: Optional[datetime]) -> Video: + other_video = self._other.parse(file, start_date) return self.__create_cached_video(other_video) def __create_cached_video(self, other_video: Video) -> Video: @@ -755,9 +758,14 @@ def parse( content = _parse_bz2(file) metadata = content[ottrk_format.METADATA][ottrk_format.VIDEO] video_file = metadata[ottrk_format.FILENAME] + metadata[ottrk_format.FILETYPE] - video = self._video_parser.parse(file.parent / video_file) + start_date = self.__parse_recorded_start_date(metadata) + video = self._video_parser.parse(file.parent / video_file, start_date) return track_ids, [video] * len(track_ids) + def __parse_recorded_start_date(self, metadata: dict) -> datetime: + start_date = metadata[ottrk_format.RECORDED_START_DATE] + return datetime.fromtimestamp(start_date) + class OtEventListParser(EventListParser): def serialize( diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 5f55db30e..ca30d553c 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -7,6 +7,7 @@ from matplotlib.axes import Axes from matplotlib.backends.backend_agg import FigureCanvasAgg from matplotlib.figure import Figure +from matplotlib.patches import Rectangle from mpl_toolkits.axes_grid1 import Divider, Size from pandas import DataFrame from PIL import Image @@ -17,12 +18,16 @@ from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.track import ( + H, PilImage, Track, TrackId, TrackIdProvider, TrackImage, TrackListObserver, + W, + X, + Y, ) from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder @@ -102,18 +107,6 @@ def plot( pass -class TrackBackgroundPlotter(Plotter): - """Plot video frame as background.""" - - def __init__(self, datastore: Datastore) -> None: - self._datastore = datastore - - def plot(self) -> Optional[TrackImage]: - if track := next(iter(self._datastore.get_all_tracks()), None): - return self._datastore.get_image_of_track(track.id) - return None - - class PlotterPrototype(Plotter): """Convenience Class to add prototype plotters to the layer structure.""" @@ -518,6 +511,56 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: ) +class TrackBoundingBoxPlotter(MatplotlibPlotterImplementation): + """Plot geometry of tracks.""" + + def __init__( + self, + data_provider: PandasDataFrameProvider, + track_view_state: TrackViewState, + alpha: float = 0.5, + ) -> None: + self._data_provider = data_provider + self._track_view_state = track_view_state + self._alpha = alpha + + def plot(self, axes: Axes) -> None: + data = self._data_provider.get_data() + if not data.empty: + self._plot_dataframe(data, axes) + + def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: + """ + Plot given tracks on the given axes with the given transparency (alpha) + + Args: + track_df (DataFrame): tracks to plot + alpha (float): transparency of the lines + axes (Axes): axes to plot on + """ + boxes_frame = track_df[track_df[track.FRAME] == self.__current_frame()] + for index, row in boxes_frame.iterrows(): + x = row[X] + y = row[Y] + width = row[W] + height = row[H] + classification = row[track.CLASSIFICATION] + axes.add_patch( + Rectangle( + xy=(x, y), + width=width, + height=height, + fc="none", + color=COLOR_PALETTE[classification], + linewidth=0.5, + alpha=0.5, + ) + ) + + def __current_frame(self) -> int: + return self._track_view_state.frame.get() + + class MatplotlibTrackPlotter(TrackPlotter): """ Implementation of the TrackPlotter interface using matplotlib. diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py index 2d3c4f03f..01432a4e0 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py @@ -1210,6 +1210,12 @@ def _update_offset( def change_track_offset_to_section_offset(self) -> None: return self._application.change_track_offset_to_section_offset() + def next_frame(self) -> None: + self._application.next_frame() + + def previous_frame(self) -> None: + self._application.previous_frame() + def validate_date(self, date: str) -> bool: return validate_date(date, DATE_FORMAT) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 31bf444c5..568278324 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -57,7 +57,7 @@ TracksNotIntersectingSelection, ) from OTAnalytics.domain.event import EventRepository, SceneEventBuilder -from OTAnalytics.domain.filter import FilterElementSettingRestorer +from OTAnalytics.domain.filter import FilterElement, FilterElementSettingRestorer from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionRepository @@ -94,6 +94,7 @@ PandasDataFrameProvider, PandasTracksOffsetProvider, PlotterPrototype, + TrackBoundingBoxPlotter, TrackGeometryPlotter, TrackStartEndPointPlotter, ) @@ -157,7 +158,7 @@ def start_gui(self) -> None: flow_state = self._create_flow_state() road_user_assigner = RoadUserAssigner() - pandas_data_provider = self._create_pandas_data_provider( + cached_track_provider = self._create_cached_pandas_track_provider( datastore, track_view_state, pulling_progressbar_builder ) layers = self._create_layers( @@ -165,7 +166,7 @@ def start_gui(self) -> None: track_view_state, flow_state, section_state, - pandas_data_provider, + cached_track_provider, road_user_assigner, ) plotter = LayeredPlotter(layers=layers) @@ -173,6 +174,8 @@ def start_gui(self) -> None: image_updater = TrackImageUpdater( datastore, track_view_state, section_state, flow_state, plotter ) + frame_updater = FrameUpdater(track_view_state) + track_view_state.filter_element.register(frame_updater.notify_filter_element) track_view_state.selected_videos.register(properties_updater.notify_videos) track_view_state.selected_videos.register(image_updater.notify_video) selected_video_updater = SelectedVideoUpdate(datastore, track_view_state) @@ -321,18 +324,25 @@ def _create_track_state(self) -> TrackState: def _create_track_view_state(self) -> TrackViewState: return TrackViewState() - def _create_pandas_data_provider( + def _create_pandas_track_offset_data_provider( + self, + track_view_state: TrackViewState, + data_provider: PandasDataFrameProvider, + ) -> PandasDataFrameProvider: + return PandasTracksOffsetProvider( + data_provider, + track_view_state=track_view_state, + ) + + def _create_cached_pandas_track_provider( self, datastore: Datastore, track_view_state: TrackViewState, progressbar: ProgressbarBuilder, ) -> PandasDataFrameProvider: dataframe_filter_builder = self._create_dataframe_filter_builder() - return PandasTracksOffsetProvider( - CachedPandasTrackProvider( - datastore, track_view_state, dataframe_filter_builder, progressbar - ), - track_view_state=track_view_state, + return CachedPandasTrackProvider( + datastore, track_view_state, dataframe_filter_builder, progressbar ) def _create_track_geometry_plotter( @@ -349,6 +359,17 @@ def _create_track_geometry_plotter( ) return PlotterPrototype(state, track_plotter) + def _create_track_bounding_box_plotter( + self, + state: TrackViewState, + pandas_data_provider: PandasDataFrameProvider, + alpha: float, + ) -> Plotter: + track_plotter = MatplotlibTrackPlotter( + TrackBoundingBoxPlotter(pandas_data_provider, state, alpha=alpha), + ) + return PlotterPrototype(state, track_plotter) + def _create_track_start_end_point_plotter( self, state: TrackViewState, @@ -465,21 +486,24 @@ def _create_layers( track_view_state: TrackViewState, flow_state: FlowState, section_state: SectionState, - pandas_data_provider: PandasDataFrameProvider, + track_data_provider: PandasDataFrameProvider, road_user_assigner: RoadUserAssigner, ) -> Sequence[PlottingLayer]: - background_image_plotter = TrackBackgroundPlotter(track_view_state, datastore) - data_provider_all_filters = FilterByClassification( - FilterByOccurrence( - pandas_data_provider, - track_view_state, - self._create_dataframe_filter_builder(), - ), + track_bounding_box_plotter = self._create_track_bounding_box_plotter( track_view_state, - self._create_dataframe_filter_builder(), + self.__create_all_filters_provider(track_view_state, track_data_provider), + alpha=0.5, + ) + track_offset_data_provider = self._create_pandas_track_offset_data_provider( + track_view_state, + track_data_provider, + ) + background_image_plotter = TrackBackgroundPlotter(track_view_state, datastore) + data_provider_all_filters = self.__create_all_filters_provider( + track_view_state, track_offset_data_provider ) data_provider_class_filter = FilterByClassification( - pandas_data_provider, + track_offset_data_provider, track_view_state, self._create_dataframe_filter_builder(), ) @@ -558,6 +582,11 @@ def _create_layers( all_tracks_layer = PlottingLayer( "Show all tracks", track_geometry_plotter, enabled=True ) + bounding_box_layer = PlottingLayer( + "Show bounding boxes of current frame", + track_bounding_box_plotter, + enabled=True, + ) highlight_tracks_intersecting_sections_layer = PlottingLayer( "Highlight tracks intersecting sections", highlight_tracks_intersecting_sections, @@ -595,6 +624,7 @@ def _create_layers( return [ background, all_tracks_layer, + bounding_box_layer, highlight_tracks_intersecting_sections_layer, highlight_tracks_not_intersecting_sections_layer, start_end_point_layer, @@ -604,6 +634,19 @@ def _create_layers( highlight_tracks_not_assigned_to_flow_layer, ] + def __create_all_filters_provider( + self, track_view_state: TrackViewState, data_provider: PandasDataFrameProvider + ) -> PandasDataFrameProvider: + return FilterByClassification( + FilterByOccurrence( + data_provider, + track_view_state, + self._create_dataframe_filter_builder(), + ), + track_view_state, + self._create_dataframe_filter_builder(), + ) + def _create_section_state(self) -> SectionState: return SectionState() @@ -672,3 +715,14 @@ def _create_export_counts( SimpleTaggerFactory(track_repository), SimpleExporterFactory(), ) + + +class FrameUpdater: + def __init__(self, state: TrackViewState) -> None: + self._state = state + + def notify_filter_element(self, filter_element: FilterElement) -> None: + end_date = filter_element.date_range.end_date + video = self._state.selected_videos.get()[0] + frame_number = video.get_frame_number_for(end_date) if end_date else 0 + self._state.frame.set(frame_number) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index 119f0ee2b..dcbfa11d8 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -1,3 +1,5 @@ +from datetime import timedelta +from math import floor from pathlib import Path from moviepy.video.io.VideoFileClip import VideoFileClip @@ -34,11 +36,19 @@ def get_frame(self, video_path: Path, index: int) -> TrackImage: except IOError as e: raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e found = None + max_frames = clip.fps * clip.duration for frame_no, np_frame in enumerate(clip.iter_frames()): - if frame_no == index: + if frame_no == (index % max_frames): found = np_frame break clip.close() if found is None: raise FrameDoesNotExistError(f"frame number '{index}' does not exist") return PilImage(Image.fromarray(found)) + + def get_frame_number_for(self, video_path: Path, delta: timedelta) -> int: + try: + clip = VideoFileClip(str(video_path.absolute())) + except IOError as e: + raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e + return floor(clip.fps * delta.total_seconds()) diff --git a/tests/OTAnalytics/plugin_video_processing/test_video_reader.py b/tests/OTAnalytics/plugin_video_processing/test_video_reader.py index fb54d1fd9..2479e58b5 100644 --- a/tests/OTAnalytics/plugin_video_processing/test_video_reader.py +++ b/tests/OTAnalytics/plugin_video_processing/test_video_reader.py @@ -1,12 +1,7 @@ from pathlib import Path -import pytest - from OTAnalytics.domain.track import PilImage -from OTAnalytics.plugin_video_processing.video_reader import ( - FrameDoesNotExistError, - MoviepyVideoReader, -) +from OTAnalytics.plugin_video_processing.video_reader import MoviepyVideoReader class TestMoviepyVideoReader: @@ -17,5 +12,5 @@ def test_get_image_possible(self, cyclist_video: Path) -> None: assert isinstance(image, PilImage) def test_get_frame_out_of_bounds(self, cyclist_video: Path) -> None: - with pytest.raises(FrameDoesNotExistError): - self.video_reader.get_frame(cyclist_video, 100) + image = self.video_reader.get_frame(cyclist_video, 100) + assert isinstance(image, PilImage) From d12ba224b3aa3b2722e4534d19c06c66e953f1a0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 09:20:07 +0200 Subject: [PATCH 02/49] Add video control to switch current frame --- .../customtkinter_gui/frame_video_control.py | 35 +++++++++++++++++++ .../plugin_ui/customtkinter_gui/gui.py | 9 +++++ 2 files changed, 44 insertions(+) create mode 100644 OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py new file mode 100644 index 000000000..79476cc2c --- /dev/null +++ b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py @@ -0,0 +1,35 @@ +from typing import Any + +from customtkinter import CTkButton, CTkFrame + +from OTAnalytics.adapter_ui.view_model import ViewModel +from OTAnalytics.plugin_ui.customtkinter_gui.constants import PADX, STICKY + + +class FrameVideoControl(CTkFrame): + def __init__(self, viewmodel: ViewModel, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._viewmodel = viewmodel + self._get_widgets() + self._place_widgets() + + def _get_widgets(self) -> None: + self.button_next_frame = CTkButton( + master=self, + text=">", + command=self._viewmodel.next_frame, + ) + self.button_previous_frame = CTkButton( + master=self, + text="<", + command=self._viewmodel.previous_frame, + ) + + def _place_widgets(self) -> None: + PADY = 10 + self.button_previous_frame.grid( + row=0, column=1, padx=PADX, pady=PADY, sticky=STICKY + ) + self.button_next_frame.grid( + row=0, column=2, padx=PADX, pady=PADY, sticky=STICKY + ) diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/gui.py b/OTAnalytics/plugin_ui/customtkinter_gui/gui.py index 8b301fa9a..65829e2c1 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/gui.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/gui.py @@ -26,6 +26,9 @@ FrameTrackPlotting, ) from OTAnalytics.plugin_ui.customtkinter_gui.frame_tracks import TracksFrame +from OTAnalytics.plugin_ui.customtkinter_gui.frame_video_control import ( + FrameVideoControl, +) from OTAnalytics.plugin_ui.customtkinter_gui.frame_videos import FrameVideos from OTAnalytics.plugin_ui.customtkinter_gui.helpers import get_widget_position from OTAnalytics.plugin_ui.customtkinter_gui.messagebox import InfoBox @@ -116,11 +119,17 @@ def __init__( master=self.ctkscrollableframe, viewmodel=self._viewmodel, ) + self._frame_video_control = FrameVideoControl( + master=self.ctkscrollableframe, viewmodel=self._viewmodel + ) self.grid_rowconfigure(1, weight=1) self.grid_columnconfigure((0, 1), weight=1) self._frame_track_plotting.grid(row=0, column=0, pady=PADY, sticky=STICKY) self._frame_filter.grid(row=0, column=1, pady=PADY, sticky=STICKY) self._frame_canvas.grid(row=1, column=0, columnspan=2, pady=PADY, sticky=STICKY) + self._frame_video_control.grid( + row=2, column=0, columnspan=2, pady=PADY, sticky=STICKY + ) class FrameNavigation(CTkFrame): From 46a2083bd18bb6ad323871d2cef1eb548e3e3e87 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 12:46:18 +0200 Subject: [PATCH 03/49] Optimize loading of frame --- .../plugin_video_processing/video_reader.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index dcbfa11d8..e89d19f85 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -31,21 +31,21 @@ def get_frame(self, video_path: Path, index: int) -> TrackImage: Returns: ndarray: the image as an multi-dimensional array. """ - try: - clip = VideoFileClip(str(video_path.absolute())) - except IOError as e: - raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e + clip = self.__get_clip(video_path) found = None max_frames = clip.fps * clip.duration - for frame_no, np_frame in enumerate(clip.iter_frames()): - if frame_no == (index % max_frames): - found = np_frame - break - clip.close() - if found is None: + if index >= max_frames: raise FrameDoesNotExistError(f"frame number '{index}' does not exist") + found = clip.get_frame(index / clip.fps) + clip.close() return PilImage(Image.fromarray(found)) + def __get_clip(self, video_path: Path) -> VideoFileClip: + try: + return VideoFileClip(str(video_path.absolute())) + except IOError as e: + raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e + def get_frame_number_for(self, video_path: Path, delta: timedelta) -> int: try: clip = VideoFileClip(str(video_path.absolute())) From 3c2d7aab1b7b866bd6b52f621b46ad179489904f Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:50:02 +0200 Subject: [PATCH 04/49] Load last frame instead of raising an exception --- OTAnalytics/plugin_video_processing/video_reader.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index e89d19f85..96c2ef97e 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -34,9 +34,8 @@ def get_frame(self, video_path: Path, index: int) -> TrackImage: clip = self.__get_clip(video_path) found = None max_frames = clip.fps * clip.duration - if index >= max_frames: - raise FrameDoesNotExistError(f"frame number '{index}' does not exist") - found = clip.get_frame(index / clip.fps) + frame_to_load = min(index, max_frames) + found = clip.get_frame(frame_to_load / clip.fps) clip.close() return PilImage(Image.fromarray(found)) From 269630b94cd7ebb65841ddc711cafbec42b5823a Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:50:17 +0200 Subject: [PATCH 05/49] Fix missing argument --- tests/OTAnalytics/plugin_parser/test_otvision_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py index ab41598e7..082d59a66 100644 --- a/tests/OTAnalytics/plugin_parser/test_otvision_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_otvision_parser.py @@ -699,7 +699,7 @@ def test_parse_to_cached_video(self, test_data_tmp_dir: Path) -> None: cached_parser = CachedVideoParser(video_parser) - parsed_video = cached_parser.parse(video_file) + parsed_video = cached_parser.parse(video_file, start_date=None) assert isinstance(parsed_video, CachedVideo) assert parsed_video.other == video From c2932e57c22db227eca79fb0b6302c992a3062de Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:51:01 +0200 Subject: [PATCH 06/49] Decouple plotter from non domain code --- OTAnalytics/application/plotting.py | 22 +++++---- .../OTAnalytics/application/test_plotting.py | 45 ++++++++++++++++++- .../track_visualization/test_track_viz.py | 30 ------------- 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index c2386d7b3..2e986bcf7 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -1,9 +1,9 @@ from abc import abstractmethod from typing import Callable, Optional, Sequence -from OTAnalytics.application.datastore import Datastore -from OTAnalytics.application.state import ObservableProperty, Plotter, TrackViewState +from OTAnalytics.application.state import ObservableProperty, Plotter from OTAnalytics.domain.track import TrackImage +from OTAnalytics.domain.video import Video class Layer: @@ -84,17 +84,23 @@ def __add(self, image: TrackImage) -> None: self._current_image = image +FrameProvider = Callable[[], int] +VideoProvider = Callable[[], list[Video]] + + class TrackBackgroundPlotter(Plotter): """Plot video frame as background.""" - def __init__(self, track_view_state: TrackViewState, datastore: Datastore) -> None: - self._track_view_state = track_view_state - self._datastore = datastore + def __init__( + self, video_provider: VideoProvider, frame_provider: FrameProvider + ) -> None: + self._video_provider = video_provider + self._frame_provider = frame_provider def plot(self) -> Optional[TrackImage]: - current_frame = self._track_view_state.frame.get() - frame_number = current_frame if current_frame else 1 - if videos := self._track_view_state.selected_videos.get(): + if videos := self._video_provider(): if len(videos) > 0: + current_frame = self._frame_provider() + frame_number = current_frame or 1 return videos[0].get_frame(frame_number) return None diff --git a/tests/OTAnalytics/application/test_plotting.py b/tests/OTAnalytics/application/test_plotting.py index 779bb10e4..25b46bb38 100644 --- a/tests/OTAnalytics/application/test_plotting.py +++ b/tests/OTAnalytics/application/test_plotting.py @@ -2,9 +2,16 @@ import pytest -from OTAnalytics.application.plotting import LayeredPlotter, PlottingLayer +from OTAnalytics.application.plotting import ( + FrameProvider, + LayeredPlotter, + PlottingLayer, + TrackBackgroundPlotter, + VideoProvider, +) from OTAnalytics.application.state import Plotter from OTAnalytics.domain.track import TrackImage +from OTAnalytics.domain.video import Video class TestLayeredPlotter: @@ -66,3 +73,39 @@ def test_plot(self, plotter: Mock) -> None: layer.set_enabled(False) layer.plot() plotter.plot.assert_called_once() + + +class TestBackgroundPlotter: + def test_plot(self) -> None: + expected_image = Mock() + single_video = Mock(spec=Video) + single_video.get_frame.return_value = expected_image + videos: list[Video] = [single_video] + video_provider = Mock(spec=VideoProvider) + video_provider.return_value = videos + frame_provider = Mock(spec=FrameProvider) + + background_plotter = TrackBackgroundPlotter( + video_provider=video_provider, frame_provider=frame_provider + ) + result = background_plotter.plot() + + video_provider.assert_called_once() + frame_provider.assert_called_once() + single_video.get_frame.assert_called_once() + assert result is not None + assert result == expected_image + + def test_plot_empty_track_repository_returns_none(self) -> None: + videos: list[Video] = [] + video_provider = Mock(spec=VideoProvider) + video_provider.return_value = videos + frame_provider = Mock(spec=FrameProvider) + background_plotter = TrackBackgroundPlotter( + video_provider=video_provider, frame_provider=frame_provider + ) + result = background_plotter.plot() + + video_provider.assert_called_once() + frame_provider.assert_not_called() + assert result is None diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 2e0c50e70..33d4b72ae 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -31,7 +31,6 @@ PandasDataFrameProvider, PandasTrackProvider, PlotterPrototype, - TrackBackgroundPlotter, TrackGeometryPlotter, TrackPlotter, TrackStartEndPointPlotter, @@ -196,35 +195,6 @@ def test_notify_update_mixed(self, track_1: Track, track_2: Track) -> None: self.check_expected_ids(provider, [track_1, track_2]) -class TestBackgroundPlotter: - def test_plot(self) -> None: - track = Mock(spec=Track).return_value - track.id = TrackId(5) - - tracks = [track] - expected_image = Mock() - datastore = Mock(spec=Datastore) - datastore.get_all_tracks.return_value = tracks - datastore.get_image_of_track.return_value = expected_image - - background_plotter = TrackBackgroundPlotter(datastore) - result = background_plotter.plot() - - datastore.get_all_tracks.assert_called_once() - datastore.get_image_of_track.assert_called_once_with(track.id) - assert result is not None - assert result == expected_image - - def test_plot_empty_track_repository_returns_none(self) -> None: - mock_datastore = Mock(spec=Datastore) - mock_datastore.get_all_tracks.return_value = [] - background_plotter = TrackBackgroundPlotter(mock_datastore) - result = background_plotter.plot() - - mock_datastore.get_all_tracks.assert_called_once() - assert result is None - - class TestTrackGeometryPlotter: @pytest.mark.parametrize( "data_frame,call_count", From 0ac26e72e47d9d4110e069deed8de447c92d156d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:51:35 +0200 Subject: [PATCH 07/49] Fix missing parameters --- .../OTAnalytics/application/test_datastore.py | 32 +++++++++++++++---- tests/OTAnalytics/domain/test_video.py | 13 ++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index 3abaac7d6..84cf28d20 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Sequence from unittest.mock import MagicMock, Mock, call @@ -32,6 +33,8 @@ from OTAnalytics.domain.types import EventType from OTAnalytics.domain.video import SimpleVideo, Video, VideoReader, VideoRepository +START_DATE = datetime(2023, 1, 1) + class MockVideoReader(VideoReader): def get_frame(self, video: Path, index: int) -> TrackImage: @@ -50,21 +53,32 @@ def height(self) -> int: return MockImage() + def get_frame_number_for(self, video_path: Path, date: timedelta) -> int: + return 0 + class TestSimpleVideo: video_reader = MockVideoReader() def test_raise_error_if_file_not_exists(self) -> None: with pytest.raises(ValueError): - SimpleVideo(video_reader=self.video_reader, path=Path("foo/bar.mp4")) + SimpleVideo( + video_reader=self.video_reader, + path=Path("foo/bar.mp4"), + start_date=START_DATE, + ) def test_init_with_valid_args(self, cyclist_video: Path) -> None: - video = SimpleVideo(video_reader=self.video_reader, path=cyclist_video) + video = SimpleVideo( + video_reader=self.video_reader, path=cyclist_video, start_date=START_DATE + ) assert video.path == cyclist_video assert video.video_reader == self.video_reader def test_get_frame_return_correct_image(self, cyclist_video: Path) -> None: - video = SimpleVideo(video_reader=self.video_reader, path=cyclist_video) + video = SimpleVideo( + video_reader=self.video_reader, path=cyclist_video, start_date=START_DATE + ) assert video.get_frame(0).as_image() == Image.fromarray( array([[1, 0], [0, 1]], int32) ) @@ -212,7 +226,9 @@ def test_load_track_file( some_track = Mock() some_track_id = TrackId(1) some_track.id = some_track_id - some_video = SimpleVideo(video_reader=Mock(), path=Path("")) + some_video = SimpleVideo( + video_reader=Mock(), path=Path(""), start_date=START_DATE + ) track_parser.parse.return_value = [some_track] track_video_parser.parse.return_value = [some_track_id], [some_video] @@ -269,11 +285,15 @@ def test_load_track_files( some_track = Mock() some_track_id = TrackId(1) some_track.id = some_track_id - some_video = SimpleVideo(video_reader=Mock(), path=Path("")) + some_video = SimpleVideo( + video_reader=Mock(), path=Path(""), start_date=START_DATE + ) other_track = Mock() other_track_id = TrackId(2) other_track.id = other_track_id - other_video = SimpleVideo(video_reader=Mock(), path=Path("")) + other_video = SimpleVideo( + video_reader=Mock(), path=Path(""), start_date=START_DATE + ) track_parser.parse.side_effect = [[some_track], [other_track]] track_video_parser.parse.side_effect = [ [[some_track_id], [some_video]], diff --git a/tests/OTAnalytics/domain/test_video.py b/tests/OTAnalytics/domain/test_video.py index dfdbf752a..5ab51fd01 100644 --- a/tests/OTAnalytics/domain/test_video.py +++ b/tests/OTAnalytics/domain/test_video.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path from unittest.mock import Mock, call, patch @@ -12,6 +13,8 @@ VideoRepository, ) +START_DATE = datetime(2023, 1, 1) + @pytest.fixture def video_reader() -> Mock: @@ -29,7 +32,9 @@ def test_resolve_relative_paths( config_path.parent.mkdir(parents=True) video_path.touch() config_path.touch() - video = SimpleVideo(path=video_path, video_reader=video_reader) + video = SimpleVideo( + path=video_path, video_reader=video_reader, start_date=START_DATE + ) result = video.to_dict(config_path) @@ -40,7 +45,9 @@ def test_resolve_relative_paths_on_different_drives( ) -> None: video_path = Mock(spec=Path) config_path = Mock(spec=Path) - video = SimpleVideo(path=video_path, video_reader=video_reader) + video = SimpleVideo( + path=video_path, video_reader=video_reader, start_date=START_DATE + ) with patch( "OTAnalytics.domain.video.splitdrive", @@ -55,7 +62,7 @@ def test_remove(self, video_reader: VideoReader, test_data_tmp_dir: Path) -> Non observer = Mock(spec=VideoListObserver) path = test_data_tmp_dir / "dummy.mp4" path.touch() - video = SimpleVideo(video_reader, path) + video = SimpleVideo(video_reader, path, start_date=START_DATE) repository = VideoRepository() repository.register_videos_observer(observer) From 1278d00277df9d42416e2eb5ecf1bf4baf32dc33 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:54:11 +0200 Subject: [PATCH 08/49] Remove duplicated code --- OTAnalytics/plugin_video_processing/video_reader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index 96c2ef97e..a044040c8 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -46,8 +46,5 @@ def __get_clip(self, video_path: Path) -> VideoFileClip: raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e def get_frame_number_for(self, video_path: Path, delta: timedelta) -> int: - try: - clip = VideoFileClip(str(video_path.absolute())) - except IOError as e: - raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e + clip = self.__get_clip(video_path) return floor(clip.fps * delta.total_seconds()) From e2e3898a0b4a17935ac1348e102b977ab0287d9d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 7 Aug 2023 14:55:57 +0200 Subject: [PATCH 09/49] Update to plotter changes --- OTAnalytics/plugin_ui/main_application.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 568278324..1a1ca822b 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -498,7 +498,9 @@ def _create_layers( track_view_state, track_data_provider, ) - background_image_plotter = TrackBackgroundPlotter(track_view_state, datastore) + background_image_plotter = TrackBackgroundPlotter( + track_view_state.selected_videos.get, track_view_state.frame.get + ) data_provider_all_filters = self.__create_all_filters_provider( track_view_state, track_offset_data_provider ) From 7e566ab8c25bdfc19c67dc175f75137cc16ab2a6 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 22 Aug 2023 13:46:40 +0200 Subject: [PATCH 10/49] Use default color palette for bounding boxes --- OTAnalytics/plugin_prototypes/track_visualization/track_viz.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index ca30d553c..a49d07844 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -544,14 +544,12 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: y = row[Y] width = row[W] height = row[H] - classification = row[track.CLASSIFICATION] axes.add_patch( Rectangle( xy=(x, y), width=width, height=height, fc="none", - color=COLOR_PALETTE[classification], linewidth=0.5, alpha=0.5, ) From 7bdff9708f24dc26b707819d40ade8fb31b3d64a Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 22 Aug 2023 13:55:12 +0200 Subject: [PATCH 11/49] Use color palette provider to determine colors --- .../plugin_prototypes/track_visualization/track_viz.py | 7 ++++++- OTAnalytics/plugin_ui/main_application.py | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index a6edcdc08..efddeac3c 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -549,15 +549,17 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: class TrackBoundingBoxPlotter(MatplotlibPlotterImplementation): - """Plot geometry of tracks.""" + """Plot bounding boxes of detections.""" def __init__( self, data_provider: PandasDataFrameProvider, + color_palette_provider: ColorPaletteProvider, track_view_state: TrackViewState, alpha: float = 0.5, ) -> None: self._data_provider = data_provider + self._color_palette_provider = color_palette_provider self._track_view_state = track_view_state self._alpha = alpha @@ -581,6 +583,8 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: y = row[Y] width = row[W] height = row[H] + classification = row[track.CLASSIFICATION] + color = self._color_palette_provider.get()[classification] axes.add_patch( Rectangle( xy=(x, y), @@ -588,6 +592,7 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: height=height, fc="none", linewidth=0.5, + color=color, alpha=0.5, ) ) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index d3217e188..79b0ccff9 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -428,10 +428,16 @@ def _create_track_bounding_box_plotter( self, state: TrackViewState, pandas_data_provider: PandasDataFrameProvider, + color_palette_provider: ColorPaletteProvider, alpha: float, ) -> Plotter: track_plotter = MatplotlibTrackPlotter( - TrackBoundingBoxPlotter(pandas_data_provider, state, alpha=alpha), + TrackBoundingBoxPlotter( + pandas_data_provider, + color_palette_provider, + state, + alpha=alpha, + ), ) return PlotterPrototype(state, track_plotter) @@ -601,6 +607,7 @@ def _create_layers( track_bounding_box_plotter = self._create_track_bounding_box_plotter( track_view_state, self.__create_all_filters_provider(track_view_state, track_data_provider), + color_palette_provider=color_palette_provider, alpha=0.5, ) track_offset_data_provider = self._create_pandas_track_offset_data_provider( From f78b8cb9e58b302432734d881ad9a826d4653822 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Sep 2023 11:15:56 +0200 Subject: [PATCH 12/49] Remove unused code --- OTAnalytics/application/plotting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index a9f083485..a6b6bc84e 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -110,7 +110,6 @@ def __init__( def plot(self) -> Optional[TrackImage]: if videos := self._video_provider(): if len(videos) > 0: - current_frame = self._frame_provider() - frame_number = current_frame or 1 + frame_number = self._frame_provider() return videos[0].get_frame(frame_number) return None From e719aa9efd6c50591dbe8c18e4505360a0ad8ae0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Sep 2023 11:17:01 +0200 Subject: [PATCH 13/49] Consider offset of frames in video and ottrk files. --- .../plugin_prototypes/track_visualization/track_viz.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index efddeac3c..8f510a0da 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -32,6 +32,9 @@ ) from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder +"""Frames start with 1 in OTVision but frames of videos are loaded zero based.""" +FRAME_OFFSET = 1 + ENCODING = "UTF-8" DPI = 100 @@ -577,7 +580,8 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: alpha (float): transparency of the lines axes (Axes): axes to plot on """ - boxes_frame = track_df[track_df[track.FRAME] == self.__current_frame()] + current_frame = self.__current_frame() + FRAME_OFFSET + boxes_frame = track_df[track_df[track.FRAME] == current_frame] for index, row in boxes_frame.iterrows(): x = row[X] y = row[Y] From 32ec727b9d9f66dc6735ac6c103adb6a7e5d3f7d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 18 Oct 2023 13:44:21 +0200 Subject: [PATCH 14/49] Fix broken test --- .../application/use_cases/test_load_track_files.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/OTAnalytics/application/use_cases/test_load_track_files.py b/tests/OTAnalytics/application/use_cases/test_load_track_files.py index 1beb488b2..f53a50d59 100644 --- a/tests/OTAnalytics/application/use_cases/test_load_track_files.py +++ b/tests/OTAnalytics/application/use_cases/test_load_track_files.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, Mock, call, patch @@ -5,6 +6,8 @@ from OTAnalytics.domain.track import TrackId from OTAnalytics.domain.video import SimpleVideo +START_DATE = datetime(2023, 1, 1) + class TestLoadTrackFile: @patch("OTAnalytics.application.use_cases.load_track_files.LoadTrackFiles.load") @@ -33,7 +36,9 @@ def test_load(self) -> None: some_track = Mock() some_track_id = TrackId("1") some_track.id = some_track_id - some_video = SimpleVideo(video_reader=Mock(), path=Path("")) + some_video = SimpleVideo( + video_reader=Mock(), path=Path(""), start_date=START_DATE + ) detection_metadata = Mock() parse_result = Mock() parse_result.tracks = [some_track] From 7333b8d3448d7d818e7912e79aac6550051c3434 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 6 Dec 2023 17:10:00 +0100 Subject: [PATCH 15/49] Use timezone information while parsing --- OTAnalytics/plugin_parser/otvision_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index 735bfa3de..bd555bdce 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -882,7 +882,7 @@ def parse( def __parse_recorded_start_date(self, metadata: dict) -> datetime: start_date = metadata[ottrk_format.RECORDED_START_DATE] - return datetime.fromtimestamp(start_date) + return datetime.fromtimestamp(start_date, tz=timezone.utc) class OtEventListParser(EventListParser): From 24998a09f44685ea0048c282ac27748ffe1a5e3b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 6 Dec 2023 17:12:13 +0100 Subject: [PATCH 16/49] Add bounding box plotter --- .../plugin_ui/visualization/visualization.py | 100 +++++++++++++----- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 58052c1ad..d47c9d990 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -45,14 +45,16 @@ FlowLayerPlotter, MatplotlibTrackPlotter, PandasDataFrameProvider, - PandasTrackProvider, PandasTracksOffsetProvider, PlotterPrototype, SectionLayerPlotter, + TrackBoundingBoxPlotter, TrackGeometryPlotter, TrackStartEndPointPlotter, ) +ALPHA_BOUNDING_BOX = 0.5 + class VisualizationBuilder: def __init__( @@ -75,7 +77,11 @@ def __init__( self._intersection_repository = intersection_repository self._event_repository = datastore._event_repository self._pandas_data_provider: Optional[PandasDataFrameProvider] = None + self._pandas_data_provider_with_offset: Optional[PandasDataFrameProvider] = None self._data_provider_all_filters: Optional[PandasDataFrameProvider] = None + self._data_provider_all_filters_with_offset: Optional[ + PandasDataFrameProvider + ] = None self._data_provider_class_filter: Optional[PandasDataFrameProvider] = None self._tracks_intersection_selected_sections: Optional[ TracksIntersectingSelectedSections @@ -103,6 +109,9 @@ def build( road_user_assigner, flow_state ) ) + + track_bounding_box_plotter = self._create_track_bounding_box_plotter() + layer_definitions = [ ("Background", background_image_plotter, True), ("Show all tracks", all_tracks_plotter, False), @@ -141,6 +150,11 @@ def build( highlight_tracks_not_assigned_to_flows_plotter, False, ), + ( + "Show bounding boxes of current frame", + track_bounding_box_plotter, + False, + ), ] return [ @@ -150,7 +164,7 @@ def build( def _create_all_tracks_plotter(self) -> Plotter: track_geometry_plotter = self._create_track_geometry_plotter( - self._get_data_provider_all_filters(), + self._get_data_provider_all_filters_with_offset(), self._color_palette_provider, alpha=0.5, enable_legend=True, @@ -231,7 +245,8 @@ def _create_highlight_tracks_assigned_to_flows_plotter( return self._create_highlight_tracks_assigned_to_flow( self._create_highlight_tracks_assigned_to_flows_factory( self._create_tracks_assigned_to_flows_filter( - self._get_data_provider_all_filters(), road_user_assigner + self._get_data_provider_all_filters_with_offset(), + road_user_assigner, ), self._color_palette_provider, alpha=1, @@ -246,7 +261,8 @@ def _create_highlight_tracks_not_assigned_to_flows_plotter( return self._create_highlight_tracks_assigned_to_flow( self._create_highlight_tracks_assigned_to_flows_factory( self._create_tracks_not_assigned_to_flows_filter( - self._get_data_provider_all_filters(), road_user_assigner + self._get_data_provider_all_filters_with_offset(), + road_user_assigner, ), self._color_palette_provider, alpha=1, @@ -258,20 +274,36 @@ def _create_highlight_tracks_not_assigned_to_flows_plotter( def _get_data_provider_class_filter(self) -> PandasDataFrameProvider: if not self._data_provider_class_filter: self._data_provider_class_filter = self._build_filter_by_classification( - self._get_pandas_data_provider() + self._get_pandas_data_provider_with_offset() ) return self._data_provider_class_filter def _get_data_provider_all_filters(self) -> PandasDataFrameProvider: if not self._data_provider_all_filters: - self._data_provider_all_filters = self._build_filter_by_classification( - self._create_filter_by_occurrence() + self._data_provider_all_filters = self._create_all_filters( + self._get_pandas_data_provider() ) return self._data_provider_all_filters - def _create_filter_by_occurrence(self) -> PandasDataFrameProvider: + def _get_data_provider_all_filters_with_offset(self) -> PandasDataFrameProvider: + if not self._data_provider_all_filters_with_offset: + self._data_provider_all_filters_with_offset = self._create_all_filters( + self._get_pandas_data_provider_with_offset() + ) + return self._data_provider_all_filters_with_offset + + def _create_all_filters( + self, data_provider: PandasDataFrameProvider + ) -> PandasDataFrameProvider: + return self._build_filter_by_classification( + self._create_filter_by_occurrence(data_provider) + ) + + def _create_filter_by_occurrence( + self, data_provider: PandasDataFrameProvider + ) -> PandasDataFrameProvider: return FilterByOccurrence( - self._get_pandas_data_provider(), + data_provider, self._track_view_state, self._create_dataframe_filter_builder(), ) @@ -283,7 +315,7 @@ def _get_tracks_not_intersecting_selected_sections_filter( self, ) -> Callable[[SectionId], PandasDataFrameProvider]: return lambda section: FilterById( - self._get_data_provider_all_filters(), + self._get_data_provider_all_filters_with_offset(), TracksNotIntersectingSelection( TracksIntersectingGivenSections( {section}, @@ -304,15 +336,14 @@ def _build_filter_by_classification( self._create_dataframe_filter_builder(), ) - def _get_pandas_data_provider(self) -> PandasDataFrameProvider: - if not self._pandas_data_provider: - cached_pandas_track_provider = self._create_pandas_track_provider( - self._pulling_progressbar_builder - ) - self._pandas_data_provider = self._wrap_pandas_track_offset_provider( - cached_pandas_track_provider + def _get_pandas_data_provider_with_offset(self) -> PandasDataFrameProvider: + if not self._pandas_data_provider_with_offset: + self._pandas_data_provider_with_offset = ( + self._wrap_pandas_track_offset_provider( + self._get_pandas_data_provider() + ) ) - return self._pandas_data_provider + return self._pandas_data_provider_with_offset def _wrap_plotter_with_cache(self, other: Plotter) -> Plotter: """ @@ -325,19 +356,19 @@ def _wrap_plotter_with_cache(self, other: Plotter) -> Plotter: self._track_view_state.track_offset.register(invalidate) return cached_plotter - def _create_pandas_track_provider( - self, progressbar: ProgressbarBuilder - ) -> PandasTrackProvider: + def _get_pandas_data_provider(self) -> PandasDataFrameProvider: dataframe_filter_builder = self._create_dataframe_filter_builder() # return PandasTrackProvider( # datastore, self._track_view_state, dataframe_filter_builder, progressbar # ) - return CachedPandasTrackProvider( - self._track_repository, - self._track_view_state, - dataframe_filter_builder, - progressbar, - ) + if not self._pandas_data_provider: + self._pandas_data_provider = CachedPandasTrackProvider( + self._track_repository, + self._track_view_state, + dataframe_filter_builder, + self._pulling_progressbar_builder, + ) + return self._pandas_data_provider def _wrap_pandas_track_offset_provider( self, other: PandasDataFrameProvider @@ -394,7 +425,7 @@ def _get_tracks_intersecting_sections_filter( self, ) -> Callable[[SectionId], PandasDataFrameProvider]: return lambda section: FilterById( - self._get_data_provider_all_filters(), + self._get_data_provider_all_filters_with_offset(), TracksIntersectingGivenSections( {section}, self._create_tracks_intersecting_sections(), @@ -584,3 +615,16 @@ def _create_tracks_intersecting_sections(self) -> TracksIntersectingSections: GetTracksWithoutSingleDetections(self._track_repository), ShapelyIntersector(), ) + + def _create_track_bounding_box_plotter( + self, + ) -> Plotter: + track_plotter = MatplotlibTrackPlotter( + TrackBoundingBoxPlotter( + self._get_data_provider_all_filters(), + self._color_palette_provider, + self._track_view_state, + alpha=ALPHA_BOUNDING_BOX, + ), + ) + return PlotterPrototype(self._track_view_state, track_plotter) From e770b3222ff1b1081670c67ecae6185f18b31fac Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 7 Dec 2023 12:57:01 +0100 Subject: [PATCH 17/49] Add fps property to video --- OTAnalytics/domain/video.py | 13 +++++++++++++ OTAnalytics/plugin_parser/otvision_parser.py | 4 ++++ OTAnalytics/plugin_video_processing/video_reader.py | 3 +++ tests/OTAnalytics/application/test_datastore.py | 3 +++ 4 files changed, 23 insertions(+) diff --git a/OTAnalytics/domain/video.py b/OTAnalytics/domain/video.py index 3c37b66ff..fb7209787 100644 --- a/OTAnalytics/domain/video.py +++ b/OTAnalytics/domain/video.py @@ -13,6 +13,10 @@ class VideoReader(ABC): + @abstractmethod + def get_fps(self, video: Path) -> float: + raise NotImplementedError + @abstractmethod def get_frame(self, video: Path, index: int) -> TrackImage: """Get frame of `video` at `index`. @@ -30,6 +34,11 @@ def get_frame_number_for(self, video_path: Path, date: timedelta) -> int: class Video(ABC): + @property + @abstractmethod + def fps(self) -> float: + raise NotImplementedError + @abstractmethod def get_path(self) -> Path: pass @@ -84,6 +93,10 @@ class SimpleVideo(Video): path: Path start_date: Optional[datetime] + @property + def fps(self) -> float: + return self.video_reader.get_fps(self.path) + def __post_init__(self) -> None: self.check_path_exists() diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index bd555bdce..82a0124af 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -824,6 +824,10 @@ class CachedVideo(Video): other: Video cache: dict[int, TrackImage] = field(default_factory=dict) + @property + def fps(self) -> float: + return self.other.fps + def get_path(self) -> Path: return self.other.get_path() diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index 63d2309b0..55accf67b 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -21,6 +21,9 @@ class FrameDoesNotExistError(Exception): class OpenCvVideoReader(VideoReader): + def get_fps(self, video_path: Path) -> float: + return self.__get_clip(video_path).get(cv2.CAP_PROP_FPS) + def get_frame(self, video_path: Path, index: int) -> TrackImage: """Get image of video at `frame`. Args: diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index d379717d9..b6dfb11c1 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -46,6 +46,9 @@ class MockVideoReader(VideoReader): + def get_fps(self, video: Path) -> float: + return 20 + def get_frame(self, video: Path, index: int) -> TrackImage: del video del index From 69ccaa17a443f6a2333430bce68b1d4be3cddd85 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 7 Dec 2023 21:07:53 +0100 Subject: [PATCH 18/49] Jump to next date range using skip time --- OTAnalytics/adapter_ui/view_model.py | 12 ++++++ OTAnalytics/application/application.py | 38 ++++++++++++++++--- OTAnalytics/application/playback.py | 7 ++++ OTAnalytics/application/plotting.py | 19 +++++++--- OTAnalytics/application/state.py | 13 +------ .../track_visualization/track_viz.py | 5 ++- .../customtkinter_gui/dummy_viewmodel.py | 10 +++++ .../customtkinter_gui/frame_video_control.py | 29 ++++++++++++-- OTAnalytics/plugin_ui/main_application.py | 15 +------- .../plugin_ui/visualization/visualization.py | 27 ++++++++++++- .../OTAnalytics/application/test_plotting.py | 21 ++++++---- 11 files changed, 149 insertions(+), 47 deletions(-) create mode 100644 OTAnalytics/application/playback.py diff --git a/OTAnalytics/adapter_ui/view_model.py b/OTAnalytics/adapter_ui/view_model.py index 4f26ece7c..20f0868da 100644 --- a/OTAnalytics/adapter_ui/view_model.py +++ b/OTAnalytics/adapter_ui/view_model.py @@ -343,3 +343,15 @@ def next_frame(self) -> None: @abstractmethod def previous_frame(self) -> None: pass + + @abstractmethod + def update_skip_time(self, seconds: int, frames: int) -> None: + raise NotImplementedError + + @abstractmethod + def get_skip_seconds(self) -> int: + raise NotImplementedError + + @abstractmethod + def get_skip_frames(self) -> int: + raise NotImplementedError diff --git a/OTAnalytics/application/application.py b/OTAnalytics/application/application.py index fa9bb271c..4c26e0825 100644 --- a/OTAnalytics/application/application.py +++ b/OTAnalytics/application/application.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Iterable, Optional @@ -450,12 +450,40 @@ def get_current_track_offset(self) -> Optional[RelativeOffsetCoordinate]: return self.track_view_state.track_offset.get() def next_frame(self) -> None: - if current := self.track_view_state.frame.get(): - self.track_view_state.frame.set(current + 1) + if videos := self.track_view_state.selected_videos.get(): + fps = videos[0].fps + filter_element = self.track_view_state.filter_element.get() + skip_time = self.track_view_state.skip_time.get() + subseconds = min(skip_time.frames, fps) / fps + current_skip = timedelta(seconds=skip_time.seconds) + timedelta( + seconds=subseconds + ) + current_date_range = filter_element.date_range + if current_date_range.start_date and current_date_range.end_date: + next_start = current_date_range.start_date + current_skip + next_end = current_date_range.end_date + current_skip + next_date_range = DateRange(next_start, next_end) + self.track_view_state.filter_element.set( + filter_element.derive_date(next_date_range) + ) def previous_frame(self) -> None: - if current := self.track_view_state.frame.get(): - self.track_view_state.frame.set(current - 1) + if videos := self.track_view_state.selected_videos.get(): + fps = videos[0].fps + filter_element = self.track_view_state.filter_element.get() + skip_time = self.track_view_state.skip_time.get() + subseconds = min(skip_time.frames, fps) / fps + current_skip = timedelta(seconds=skip_time.seconds) + timedelta( + seconds=subseconds + ) + current_date_range = filter_element.date_range + if current_date_range.start_date and current_date_range.end_date: + next_start = current_date_range.start_date - current_skip + next_end = current_date_range.end_date - current_skip + next_date_range = DateRange(next_start, next_end) + self.track_view_state.filter_element.set( + filter_element.derive_date(next_date_range) + ) def update_date_range_tracks_filter(self, date_range: DateRange) -> None: """Update the date range of the track filter. diff --git a/OTAnalytics/application/playback.py b/OTAnalytics/application/playback.py new file mode 100644 index 000000000..d84bb4c96 --- /dev/null +++ b/OTAnalytics/application/playback.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SkipTime: + seconds: int + frames: int diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index ccdaa1518..6dfbd7549 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -1,4 +1,5 @@ -from abc import abstractmethod +from abc import ABC, abstractmethod +from datetime import datetime from typing import Any, Callable, Generic, Iterable, Optional, Sequence, TypeVar from OTAnalytics.application.state import ( @@ -98,7 +99,12 @@ def __add(self, image: TrackImage) -> None: self._current_image = image -FrameProvider = Callable[[], int] +class VisualizationTimeProvider(ABC): + @abstractmethod + def get_time(self) -> datetime: + raise NotImplementedError + + VideoProvider = Callable[[], list[Video]] @@ -106,15 +112,18 @@ class TrackBackgroundPlotter(Plotter): """Plot video frame as background.""" def __init__( - self, video_provider: VideoProvider, frame_provider: FrameProvider + self, + video_provider: VideoProvider, + visualization_time_provider: VisualizationTimeProvider, ) -> None: self._video_provider = video_provider - self._frame_provider = frame_provider + self._visualization_time_provider = visualization_time_provider def plot(self) -> Optional[TrackImage]: if videos := self._video_provider(): if len(videos) > 0: - frame_number = self._frame_provider() + visualization_time = self._visualization_time_provider.get_time() + frame_number = videos[0].get_frame_number_for(visualization_time) return videos[0].get_frame(frame_number) return None diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index 68f4c6fa0..52e73eab5 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -4,6 +4,7 @@ from OTAnalytics.application.config import DEFAULT_TRACK_OFFSET from OTAnalytics.application.datastore import Datastore, VideoMetadata +from OTAnalytics.application.playback import SkipTime from OTAnalytics.application.use_cases.section_repository import GetSectionsById from OTAnalytics.domain.date import DateRange from OTAnalytics.domain.event import EventRepositoryEvent @@ -187,7 +188,7 @@ def __init__(self) -> None: self.selected_videos: ObservableProperty[list[Video]] = ObservableProperty[ list[Video] ](default=[]) - self.frame = ObservableProperty[int](1) + self.skip_time = ObservableProperty[SkipTime](SkipTime(1, 0)) def reset(self) -> None: """Reset to default settings.""" @@ -324,7 +325,6 @@ def __init__( self._flow_state = flow_state self._plotter = plotter self._track_view_state.track_offset.register(self._notify_track_offset) - self._track_view_state.frame.register(self._notify_frame) self._track_view_state.filter_element.register(self._notify_filter_element) self._section_state.selected_sections.register(self._notify_section_selection) self._flow_state.selected_flows.register(self._notify_flow_changed) @@ -356,15 +356,6 @@ def _notify_track_offset(self, offset: Optional[RelativeOffsetCoordinate]) -> No """ self._update() - def _notify_frame(self, frame: Optional[int]) -> None: - """ - Will update the image according to changes of the track offset property. - - Args: - offset (Optional[RelativeOffsetCoordinate]): current value - """ - self._update() - def _notify_filter_element(self, _: FilterElement) -> None: """ Will update the image according to changes of the filter element. diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 148d35b03..f3fc09d5e 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -727,7 +727,10 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: ) def __current_frame(self) -> int: - return self._track_view_state.frame.get() + if end_date := self._track_view_state.filter_element.get().date_range.end_date: + video = self._track_view_state.selected_videos.get()[0] + return video.get_frame_number_for(end_date) + return 0 class MatplotlibTrackPlotter(TrackPlotter): diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py index 7073ebe50..ea082ee2f 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py @@ -48,6 +48,7 @@ ) from OTAnalytics.application.datastore import FlowParser, NoSectionsToSave from OTAnalytics.application.logger import logger +from OTAnalytics.application.playback import SkipTime from OTAnalytics.application.use_cases.config import MissingDate from OTAnalytics.application.use_cases.cut_tracks_with_sections import CutTracksDto from OTAnalytics.application.use_cases.export_events import ( @@ -1596,3 +1597,12 @@ def on_tracks_cut(self, cut_tracks_dto: CutTracksDto) -> None: def set_analysis_frame(self, frame: AbstractFrame) -> None: self._frame_analysis = frame + + def update_skip_time(self, seconds: int, frames: int) -> None: + self._application.track_view_state.skip_time.set(SkipTime(seconds, frames)) + + def get_skip_seconds(self) -> int: + return self._application.track_view_state.skip_time.get().seconds + + def get_skip_frames(self) -> int: + return self._application.track_view_state.skip_time.get().frames diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py index 79476cc2c..a5ed291b9 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py @@ -1,6 +1,7 @@ +import tkinter from typing import Any -from customtkinter import CTkButton, CTkFrame +from customtkinter import CTkButton, CTkEntry, CTkFrame, CTkLabel from OTAnalytics.adapter_ui.view_model import ViewModel from OTAnalytics.plugin_ui.customtkinter_gui.constants import PADX, STICKY @@ -10,8 +11,11 @@ class FrameVideoControl(CTkFrame): def __init__(self, viewmodel: ViewModel, **kwargs: Any) -> None: super().__init__(**kwargs) self._viewmodel = viewmodel + self._seconds = tkinter.IntVar(value=viewmodel.get_skip_seconds()) + self._frames = tkinter.IntVar(value=viewmodel.get_skip_frames()) self._get_widgets() self._place_widgets() + self._wire_widgets() def _get_widgets(self) -> None: self.button_next_frame = CTkButton( @@ -24,12 +28,31 @@ def _get_widgets(self) -> None: text="<", command=self._viewmodel.previous_frame, ) + self._label_seconds = CTkLabel( + master=self, text="Seconds", anchor="e", justify="right" + ) + self._label_frames = CTkLabel( + master=self, text="Frames", anchor="e", justify="right" + ) + self._entry_seconds = CTkEntry(master=self, textvariable=self._seconds) + self._entry_frames = CTkEntry(master=self, textvariable=self._frames) def _place_widgets(self) -> None: PADY = 10 self.button_previous_frame.grid( - row=0, column=1, padx=PADX, pady=PADY, sticky=STICKY + row=0, column=1, rowspan=2, padx=PADX, pady=PADY, sticky=STICKY ) + self._label_seconds.grid(row=0, column=2, padx=PADX, pady=PADY, sticky=STICKY) + self._entry_seconds.grid(row=0, column=3, padx=PADX, pady=PADY, sticky=STICKY) + self._label_frames.grid(row=1, column=2, padx=PADX, pady=PADY, sticky=STICKY) + self._entry_frames.grid(row=1, column=3, padx=PADX, pady=PADY, sticky=STICKY) self.button_next_frame.grid( - row=0, column=2, padx=PADX, pady=PADY, sticky=STICKY + row=0, column=4, rowspan=2, padx=PADX, pady=PADY, sticky=STICKY ) + + def _wire_widgets(self) -> None: + self._seconds.trace_add("write", callback=self._update_skip_time) + self._frames.trace_add("write", callback=self._update_skip_time) + + def _update_skip_time(self, name: str, other: str, mode: str) -> None: + self._viewmodel.update_skip_time(self._seconds.get(), self._frames.get()) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 948713329..6352d639b 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -94,7 +94,7 @@ from OTAnalytics.application.use_cases.update_project import ProjectUpdater from OTAnalytics.application.use_cases.video_repository import ClearAllVideos from OTAnalytics.domain.event import EventRepository, SceneEventBuilder -from OTAnalytics.domain.filter import FilterElement, FilterElementSettingRestorer +from OTAnalytics.domain.filter import FilterElementSettingRestorer from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.intersect import IntersectImplementation from OTAnalytics.domain.progress import ProgressbarBuilder @@ -248,8 +248,6 @@ def start_gui(self) -> None: image_updater = TrackImageUpdater( datastore, track_view_state, section_state, flow_state, plotter ) - frame_updater = FrameUpdater(track_view_state) - track_view_state.filter_element.register(frame_updater.notify_filter_element) track_view_state.selected_videos.register(properties_updater.notify_videos) track_view_state.selected_videos.register(image_updater.notify_video) selected_video_updater = SelectedVideoUpdate(datastore, track_view_state) @@ -848,14 +846,3 @@ def _create_video_repository(self) -> VideoRepository: def _create_track_to_video_repository(self) -> TrackToVideoRepository: return TrackToVideoRepository() - - -class FrameUpdater: - def __init__(self, state: TrackViewState) -> None: - self._state = state - - def notify_filter_element(self, filter_element: FilterElement) -> None: - end_date = filter_element.date_range.end_date - video = self._state.selected_videos.get()[0] - frame_number = video.get_frame_number_for(end_date) if end_date else 0 - self._state.frame.set(frame_number) diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index d47c9d990..479fdd23e 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Callable, Optional, Sequence from OTAnalytics.application.analysis.intersect import TracksIntersectingSections @@ -7,6 +8,7 @@ CachedPlotter, PlottingLayer, TrackBackgroundPlotter, + VisualizationTimeProvider, ) from OTAnalytics.application.state import ( FlowState, @@ -53,9 +55,28 @@ TrackStartEndPointPlotter, ) +LONG_IN_THE_PAST = datetime( + year=1970, + month=1, + day=1, + hour=0, + minute=0, + second=0, + tzinfo=timezone.utc, +) ALPHA_BOUNDING_BOX = 0.5 +class FilterEndDateProvider(VisualizationTimeProvider): + def __init__(self, state: TrackViewState) -> None: + self._state = state + + def get_time(self) -> datetime: + if end_date := self._state.filter_element.get().date_range.end_date: + return end_date + return LONG_IN_THE_PAST + + class VisualizationBuilder: def __init__( self, @@ -76,6 +97,9 @@ def __init__( self._flow_repository = datastore._flow_repository self._intersection_repository = intersection_repository self._event_repository = datastore._event_repository + self._visualization_time_provider: VisualizationTimeProvider = ( + FilterEndDateProvider(track_view_state) + ) self._pandas_data_provider: Optional[PandasDataFrameProvider] = None self._pandas_data_provider_with_offset: Optional[PandasDataFrameProvider] = None self._data_provider_all_filters: Optional[PandasDataFrameProvider] = None @@ -96,7 +120,8 @@ def build( road_user_assigner: RoadUserAssigner, ) -> Sequence[PlottingLayer]: background_image_plotter = TrackBackgroundPlotter( - self._track_view_state.selected_videos.get, self._track_view_state.frame.get + self._track_view_state.selected_videos.get, + self._visualization_time_provider, ) all_tracks_plotter = self._create_all_tracks_plotter() highlight_tracks_assigned_to_flows_plotter = ( diff --git a/tests/OTAnalytics/application/test_plotting.py b/tests/OTAnalytics/application/test_plotting.py index 25b46bb38..a2f92828b 100644 --- a/tests/OTAnalytics/application/test_plotting.py +++ b/tests/OTAnalytics/application/test_plotting.py @@ -1,13 +1,14 @@ +from datetime import datetime from unittest.mock import Mock, call import pytest from OTAnalytics.application.plotting import ( - FrameProvider, LayeredPlotter, PlottingLayer, TrackBackgroundPlotter, VideoProvider, + VisualizationTimeProvider, ) from OTAnalytics.application.state import Plotter from OTAnalytics.domain.track import TrackImage @@ -79,19 +80,24 @@ class TestBackgroundPlotter: def test_plot(self) -> None: expected_image = Mock() single_video = Mock(spec=Video) + single_video.get_frame_number_for.return_value = 0 single_video.get_frame.return_value = expected_image videos: list[Video] = [single_video] video_provider = Mock(spec=VideoProvider) video_provider.return_value = videos - frame_provider = Mock(spec=FrameProvider) + some_time = datetime(2023, 1, 1, 0, 0) + visualization_time_provider = Mock(spec=VisualizationTimeProvider) + visualization_time_provider.get_time.return_value = some_time background_plotter = TrackBackgroundPlotter( - video_provider=video_provider, frame_provider=frame_provider + video_provider=video_provider, + visualization_time_provider=visualization_time_provider, ) result = background_plotter.plot() video_provider.assert_called_once() - frame_provider.assert_called_once() + visualization_time_provider.get_time.assert_called_once() + single_video.get_frame_number_for.assert_called_with(some_time) single_video.get_frame.assert_called_once() assert result is not None assert result == expected_image @@ -100,12 +106,13 @@ def test_plot_empty_track_repository_returns_none(self) -> None: videos: list[Video] = [] video_provider = Mock(spec=VideoProvider) video_provider.return_value = videos - frame_provider = Mock(spec=FrameProvider) + visualization_time_provider = Mock(spec=VisualizationTimeProvider) background_plotter = TrackBackgroundPlotter( - video_provider=video_provider, frame_provider=frame_provider + video_provider=video_provider, + visualization_time_provider=visualization_time_provider, ) result = background_plotter.plot() video_provider.assert_called_once() - frame_provider.assert_not_called() + visualization_time_provider.get_time.assert_not_called() assert result is None From afbc8f4ae7b3489f37e72aa2f2e09f360cdabebf Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 7 Dec 2023 21:24:17 +0100 Subject: [PATCH 19/49] Enable and disable skip buttons --- OTAnalytics/adapter_ui/view_model.py | 4 ++++ .../customtkinter_gui/dummy_viewmodel.py | 16 +++++++++++++++ .../customtkinter_gui/frame_video_control.py | 20 +++++++++++++------ OTAnalytics/plugin_ui/main_application.py | 3 +++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/adapter_ui/view_model.py b/OTAnalytics/adapter_ui/view_model.py index 20f0868da..f0d03d318 100644 --- a/OTAnalytics/adapter_ui/view_model.py +++ b/OTAnalytics/adapter_ui/view_model.py @@ -355,3 +355,7 @@ def get_skip_seconds(self) -> int: @abstractmethod def get_skip_frames(self) -> int: raise NotImplementedError + + @abstractmethod + def set_video_control_frame(self, frame: AbstractFrame) -> None: + raise NotImplementedError diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py index ea082ee2f..2172202f6 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/dummy_viewmodel.py @@ -191,6 +191,7 @@ def __init__( self._frame_tracks: Optional[AbstractFrameTracks] = None self._frame_videos: Optional[AbstractFrame] = None self._frame_canvas: Optional[AbstractFrameCanvas] = None + self._frame_video_control: Optional[AbstractFrame] = None self._frame_sections: Optional[AbstractFrame] = None self._frame_flows: Optional[AbstractFrame] = None self._frame_filter: Optional[AbstractFrameFilter] = None @@ -1606,3 +1607,18 @@ def get_skip_seconds(self) -> int: def get_skip_frames(self) -> int: return self._application.track_view_state.skip_time.get().frames + + def set_video_control_frame(self, frame: AbstractFrame) -> None: + self._frame_video_control = frame + self.notify_filter_element_change( + self._application.track_view_state.filter_element.get() + ) + + def notify_filter_element_change(self, filter_element: FilterElement) -> None: + if not self._frame_video_control: + raise MissingInjectedInstanceError("Frame video control missing") + filter_element_is_set = ( + filter_element.date_range.start_date is not None + and filter_element.date_range.end_date is not None + ) + self._frame_video_control.set_enabled_general_buttons(filter_element_is_set) diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py index a5ed291b9..1ab330987 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/frame_video_control.py @@ -1,13 +1,14 @@ import tkinter from typing import Any -from customtkinter import CTkButton, CTkEntry, CTkFrame, CTkLabel +from customtkinter import CTkButton, CTkEntry, CTkLabel from OTAnalytics.adapter_ui.view_model import ViewModel +from OTAnalytics.plugin_ui.customtkinter_gui.abstract_ctk_frame import AbstractCTkFrame from OTAnalytics.plugin_ui.customtkinter_gui.constants import PADX, STICKY -class FrameVideoControl(CTkFrame): +class FrameVideoControl(AbstractCTkFrame): def __init__(self, viewmodel: ViewModel, **kwargs: Any) -> None: super().__init__(**kwargs) self._viewmodel = viewmodel @@ -16,14 +17,18 @@ def __init__(self, viewmodel: ViewModel, **kwargs: Any) -> None: self._get_widgets() self._place_widgets() self._wire_widgets() + self.introduce_to_viewmodel() + + def introduce_to_viewmodel(self) -> None: + self._viewmodel.set_video_control_frame(self) def _get_widgets(self) -> None: - self.button_next_frame = CTkButton( + self._button_next_frame = CTkButton( master=self, text=">", command=self._viewmodel.next_frame, ) - self.button_previous_frame = CTkButton( + self._button_previous_frame = CTkButton( master=self, text="<", command=self._viewmodel.previous_frame, @@ -39,14 +44,14 @@ def _get_widgets(self) -> None: def _place_widgets(self) -> None: PADY = 10 - self.button_previous_frame.grid( + self._button_previous_frame.grid( row=0, column=1, rowspan=2, padx=PADX, pady=PADY, sticky=STICKY ) self._label_seconds.grid(row=0, column=2, padx=PADX, pady=PADY, sticky=STICKY) self._entry_seconds.grid(row=0, column=3, padx=PADX, pady=PADY, sticky=STICKY) self._label_frames.grid(row=1, column=2, padx=PADX, pady=PADY, sticky=STICKY) self._entry_frames.grid(row=1, column=3, padx=PADX, pady=PADY, sticky=STICKY) - self.button_next_frame.grid( + self._button_next_frame.grid( row=0, column=4, rowspan=2, padx=PADX, pady=PADY, sticky=STICKY ) @@ -54,5 +59,8 @@ def _wire_widgets(self) -> None: self._seconds.trace_add("write", callback=self._update_skip_time) self._frames.trace_add("write", callback=self._update_skip_time) + def get_general_buttons(self) -> list[CTkButton]: + return [self._button_previous_frame, self._button_next_frame] + def _update_skip_time(self, name: str, other: str, mode: str) -> None: self._viewmodel.update_skip_time(self._seconds.get(), self._frames.get()) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 6352d639b..33baad70c 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -425,6 +425,9 @@ def start_gui(self) -> None: application.action_state.action_running.register( dummy_viewmodel._notify_action_running_state ) + application.track_view_state.filter_element.register( + dummy_viewmodel.notify_filter_element_change + ) # TODO: Refactor observers - move registering to subjects happening in # constructor dummy_viewmodel # cut_tracks_intersecting_section.register( From d2cfb6fc162702c5d28a7faed29ccf9d41b2dee3 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 7 Dec 2023 21:29:42 +0100 Subject: [PATCH 20/49] Fix incorrect calculation of frame number --- OTAnalytics/plugin_video_processing/video_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index 55accf67b..a113ec0f8 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -52,5 +52,5 @@ def __get_clip(video_path: Path) -> VideoCapture: def get_frame_number_for(self, video_path: Path, delta: timedelta) -> int: clip = self.__get_clip(video_path) - total_frames = int(clip.get(cv2.CAP_PROP_FRAME_COUNT)) + total_frames = int(clip.get(cv2.CAP_PROP_FPS)) return floor(total_frames * delta.total_seconds()) From 8e31564c85644071ea34745cb9b8bc53331b86f4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 7 Dec 2023 21:33:03 +0100 Subject: [PATCH 21/49] Test reading correct frame number --- .../plugin_video_processing/test_video_reader.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/OTAnalytics/plugin_video_processing/test_video_reader.py b/tests/OTAnalytics/plugin_video_processing/test_video_reader.py index 51b1c1f7a..2507833f6 100644 --- a/tests/OTAnalytics/plugin_video_processing/test_video_reader.py +++ b/tests/OTAnalytics/plugin_video_processing/test_video_reader.py @@ -1,3 +1,4 @@ +from datetime import timedelta from pathlib import Path from OTAnalytics.domain.track import PilImage @@ -14,3 +15,10 @@ def test_get_image_possible(self, cyclist_video: Path) -> None: def test_get_frame_out_of_bounds(self, cyclist_video: Path) -> None: image = self.video_reader.get_frame(cyclist_video, 100) assert isinstance(image, PilImage) + + def test_get_frame_number_for(self, cyclist_video: Path) -> None: + delta = timedelta(seconds=1) + + frame_number = self.video_reader.get_frame_number_for(cyclist_video, delta) + + assert frame_number == 20 From 01b68eeb34b627a14805808ab25801b6f5b1ea1b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 11 Dec 2023 12:02:24 +0100 Subject: [PATCH 22/49] Store seconds of track to fix visualization of tracks from multiple files --- OTAnalytics/domain/track.py | 1 + .../track_visualization/track_viz.py | 19 +++++++++++++++++-- .../track_visualization/test_track_viz.py | 17 +++++++++++------ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index a473a75c4..a2253215f 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -20,6 +20,7 @@ H: str = "h" FRAME: str = "frame" OCCURRENCE: str = "occurrence" +SECONDS: str = "seconds" INTERPOLATED_DETECTION: str = "interpolated_detection" TRACK_ID: str = "track_id" VIDEO_NAME: str = "video_name" diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index f3fc09d5e..f5a4b3b90 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -1,5 +1,6 @@ import random from abc import ABC, abstractmethod +from datetime import datetime from typing import Iterable, Optional import numpy @@ -381,7 +382,9 @@ def __init__( def get_data(self) -> DataFrame: tracks = self._track_repository.get_all() if isinstance(tracks, PandasTrackDataset): - return tracks.as_dataframe() + data = tracks.as_dataframe() + data[track.SECONDS] = data[track.OCCURRENCE].dt.floor("s") + return data track_list = tracks.as_list() if not track_list: return DataFrame() @@ -409,7 +412,11 @@ def _convert_tracks(self, tracks: Iterable[Track]) -> DataFrame: ] = current_track.classification prepared.append(detection_dict) - return self._sort_tracks(DataFrame(prepared)) + if len(prepared) == 0: + return DataFrame() + data = DataFrame(prepared) + data[track.SECONDS] = data[track.OCCURRENCE].dt.floor("s") + return self._sort_tracks(data) def _sort_tracks(self, track_df: DataFrame) -> DataFrame: """Sort the given dataframe by trackId and frame, @@ -706,6 +713,9 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: axes (Axes): axes to plot on """ current_frame = self.__current_frame() + FRAME_OFFSET + current_second = self.__current_second(track_df) + if current_second: + track_df = track_df[track_df[track.SECONDS] == current_second] boxes_frame = track_df[track_df[track.FRAME] == current_frame] for index, row in boxes_frame.iterrows(): x = row[X] @@ -732,6 +742,11 @@ def __current_frame(self) -> int: return video.get_frame_number_for(end_date) return 0 + def __current_second(self, data: DataFrame) -> Optional[datetime]: + if end_date := self._track_view_state.filter_element.get().date_range.end_date: + return end_date.replace(microsecond=0) + return data[track.SECONDS].min() + class MatplotlibTrackPlotter(TrackPlotter): """ diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 529efb023..50b1ec555 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta, timezone from unittest.mock import Mock, patch import pytest @@ -123,6 +123,11 @@ def track_2(self) -> Track: def set_up_track(self, id: str) -> Track: """Create a dummy track with the given id and 5 car detections.""" + first_detection_occurrence = datetime(2020, 1, 1, 0, 0, tzinfo=timezone.utc) + second_occurrence = first_detection_occurrence + timedelta(seconds=1) + third_occurrence = second_occurrence + timedelta(seconds=1) + fourth_occurrence = third_occurrence + timedelta(seconds=1) + fives_occurrence = fourth_occurrence + timedelta(seconds=1) t_id = TrackId(id) detections: list[Detection] = [ PythonDetection( @@ -133,7 +138,7 @@ def set_up_track(self, id: str) -> Track: 2, 7, 1, - datetime.min, + first_detection_occurrence, False, t_id, "video_name", @@ -146,7 +151,7 @@ def set_up_track(self, id: str) -> Track: 2, 7, 2, - datetime.min, + second_occurrence, False, t_id, "video_name", @@ -159,7 +164,7 @@ def set_up_track(self, id: str) -> Track: 2, 7, 3, - datetime.min, + third_occurrence, False, t_id, "video_name", @@ -172,7 +177,7 @@ def set_up_track(self, id: str) -> Track: 2, 7, 4, - datetime.min, + fourth_occurrence, False, t_id, "video_name", @@ -185,7 +190,7 @@ def set_up_track(self, id: str) -> Track: 2, 7, 5, - datetime.min, + fives_occurrence, False, t_id, "video_name", From 32448ff2300c20b5d443251ca8e2de3fe1d95310 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 11 Dec 2023 18:46:18 +0100 Subject: [PATCH 23/49] Clean up code and use correct data providers --- .../track_visualization/track_viz.py | 12 +++--------- .../plugin_ui/visualization/visualization.py | 16 +++++++++++----- .../track_visualization/test_track_viz.py | 8 ++------ 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index f5a4b3b90..42abeb61d 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -370,12 +370,10 @@ class PandasTrackProvider(PandasDataFrameProvider): def __init__( self, track_repository: TrackRepository, - track_view_state: TrackViewState, filter_builder: DataFrameFilterBuilder, progressbar: ProgressbarBuilder, ) -> None: self._track_repository = track_repository - self._track_view_state = track_view_state self._filter_builder = filter_builder self._progressbar = progressbar @@ -464,13 +462,10 @@ class CachedPandasTrackProvider(PandasTrackProvider, TrackListObserver): def __init__( self, track_repository: TrackRepository, - track_view_state: TrackViewState, filter_builder: DataFrameFilterBuilder, progressbar: ProgressbarBuilder, ) -> None: - super().__init__( - track_repository, track_view_state, filter_builder, progressbar - ) + super().__init__(track_repository, filter_builder, progressbar) track_repository.register_tracks_observer(self) self._cache_df: DataFrame = DataFrame() @@ -714,9 +709,8 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: """ current_frame = self.__current_frame() + FRAME_OFFSET current_second = self.__current_second(track_df) - if current_second: - track_df = track_df[track_df[track.SECONDS] == current_second] - boxes_frame = track_df[track_df[track.FRAME] == current_frame] + timed_df = track_df[track_df[track.SECONDS] == current_second] + boxes_frame = timed_df[timed_df[track.FRAME] == current_frame] for index, row in boxes_frame.iterrows(): x = row[X] y = row[Y] diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 479fdd23e..d0febb324 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -223,7 +223,7 @@ def _create_start_end_point_intersecting_sections_plotter(self) -> Plotter: start_end_points_intersecting = self._create_cached_section_layer_plotter( self._create_start_end_point_intersecting_section_factory( self._create_tracks_start_end_point_intersecting_given_sections_filter( - self._get_data_provider_class_filter(), + self._get_data_provider_class_filter_with_offset(), self._create_tracks_intersecting_sections(), self._create_get_sections_by_id(), ), @@ -237,7 +237,7 @@ def _create_start_end_point_intersecting_sections_plotter(self) -> Plotter: def _create_start_end_point_not_intersection_sections_plotter(self) -> Plotter: section_filter = ( self._create_tracks_start_end_point_not_intersecting_given_sections_filter( - self._get_data_provider_class_filter(), + self._get_data_provider_class_filter_with_offset(), self._create_tracks_intersecting_sections(), self._create_get_sections_by_id(), ) @@ -254,7 +254,7 @@ def _create_start_end_point_not_intersection_sections_plotter(self) -> Plotter: def _create_start_end_point_plotter(self) -> Plotter: track_start_end_point_plotter = self._create_track_start_end_point_plotter( self._create_track_start_end_point_data_provider( - self._get_data_provider_class_filter() + self._get_data_provider_class_filter_with_offset() ), self._color_palette_provider, enable_legend=False, @@ -297,6 +297,13 @@ def _create_highlight_tracks_not_assigned_to_flows_plotter( ) def _get_data_provider_class_filter(self) -> PandasDataFrameProvider: + if not self._data_provider_class_filter: + self._data_provider_class_filter = self._build_filter_by_classification( + self._get_pandas_data_provider() + ) + return self._data_provider_class_filter + + def _get_data_provider_class_filter_with_offset(self) -> PandasDataFrameProvider: if not self._data_provider_class_filter: self._data_provider_class_filter = self._build_filter_by_classification( self._get_pandas_data_provider_with_offset() @@ -389,7 +396,6 @@ def _get_pandas_data_provider(self) -> PandasDataFrameProvider: if not self._pandas_data_provider: self._pandas_data_provider = CachedPandasTrackProvider( self._track_repository, - self._track_view_state, dataframe_filter_builder, self._pulling_progressbar_builder, ) @@ -646,7 +652,7 @@ def _create_track_bounding_box_plotter( ) -> Plotter: track_plotter = MatplotlibTrackPlotter( TrackBoundingBoxPlotter( - self._get_data_provider_all_filters(), + self._get_data_provider_class_filter(), self._color_palette_provider, self._track_view_state, alpha=ALPHA_BOUNDING_BOX, diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 50b1ec555..88efd7a70 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -97,12 +97,10 @@ class TestPandasTrackProvider: def test_get_data_empty_track_repository(self) -> None: track_repository = Mock(spec=TrackRepository) track_repository.get_all.return_value = PythonTrackDataset.from_list([]) - track_view_state = Mock(spec=TrackViewState).return_value - track_view_state.track_offset.get.return_value = RelativeOffsetCoordinate(0, 0) filter_builder = Mock(FilterBuilder) provider = PandasTrackProvider( - track_repository, track_view_state, filter_builder, NoProgressbarBuilder() + track_repository, filter_builder, NoProgressbarBuilder() ) result = provider.get_data() @@ -209,11 +207,9 @@ def set_up_provider( track_repository = Mock(spec=TrackRepository) track_repository.get_for.side_effect = query_tracks - track_view_state = Mock(spec=TrackViewState).return_value - track_view_state.track_offset.get.return_value = RelativeOffsetCoordinate(0, 0) filter_builder = Mock(spec=FilterBuilder) provider = CachedPandasTrackProvider( - track_repository, track_view_state, filter_builder, NoProgressbarBuilder() + track_repository, filter_builder, NoProgressbarBuilder() ) assert provider._cache_df.empty From c0d528da5c3c8f41ef1e56033006e595bf890f6c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 11 Dec 2023 18:48:36 +0100 Subject: [PATCH 24/49] Reuse existing code --- OTAnalytics/plugin_video_processing/video_reader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_video_processing/video_reader.py b/OTAnalytics/plugin_video_processing/video_reader.py index a113ec0f8..02c3dfc2f 100644 --- a/OTAnalytics/plugin_video_processing/video_reader.py +++ b/OTAnalytics/plugin_video_processing/video_reader.py @@ -51,6 +51,4 @@ def __get_clip(video_path: Path) -> VideoCapture: raise InvalidVideoError(f"{str(video_path)} is not a valid video") from e def get_frame_number_for(self, video_path: Path, delta: timedelta) -> int: - clip = self.__get_clip(video_path) - total_frames = int(clip.get(cv2.CAP_PROP_FPS)) - return floor(total_frames * delta.total_seconds()) + return floor(self.get_fps(video_path) * delta.total_seconds()) From 76775c53a3b0920d8c257bf0eb6f5f05494bd3aa Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 16:43:50 +0100 Subject: [PATCH 25/49] Change implementation of videos metadata to get metadata by path and create use case to get current frame information --- OTAnalytics/application/plotting.py | 27 ++++++++++++++++++++++++++- OTAnalytics/application/state.py | 18 +++++++++++++----- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index 6dfbd7549..5eff40587 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -6,9 +6,11 @@ ObservableOptionalProperty, ObservableProperty, Plotter, + TrackViewState, + VideosMetadata, ) from OTAnalytics.domain.track import TrackImage -from OTAnalytics.domain.video import Video +from OTAnalytics.domain.video import Video, VideoRepository class Layer: @@ -255,3 +257,26 @@ def _handle_remove(self, entities: Iterable[ENTITY]) -> None: for entity in entities: del self._plotter_mapping[entity] del self._layer_mapping[entity] + + +class GetCurrentFrame: + def __init__( + self, + state: TrackViewState, + videos_metadata: VideosMetadata, + video_repository: VideoRepository, + ) -> None: + self._state = state + self._videos_metadata = videos_metadata + self._video_repository = video_repository + + def get_frame_number(self) -> int: + if end_date := self._state.filter_element.get().date_range.end_date: + video = self._state.selected_videos.get()[0] + return video.get_frame_number_for(end_date) + return 0 + + def get_second(self) -> Optional[datetime]: + if end_date := self._state.filter_element.get().date_range.end_date: + return end_date.replace(microsecond=0) + return self._videos_metadata.first_video_start diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index 52e73eab5..ef88faa4f 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -408,19 +408,27 @@ def _update_image(self) -> None: class VideosMetadata: def __init__(self) -> None: - self._metadata: list[VideoMetadata] = [] + self._metadata: dict[str, VideoMetadata] = {} + self._first_video_start: Optional[datetime] = None + self._last_video_end: Optional[datetime] = None def update(self, metadata: VideoMetadata) -> None: - self._metadata.append(metadata) - self._metadata.sort(key=lambda current: current.start) + self._metadata[metadata.path] = metadata + self._update_start_end_by(metadata) + + def _update_start_end_by(self, metadata: VideoMetadata) -> None: + if (not self._first_video_start) or metadata.start < self._first_video_start: + self._first_video_start = metadata.start + if (not self._last_video_end) or metadata.end > self._last_video_end: + self._last_video_end = metadata.end @property def first_video_start(self) -> Optional[datetime]: - return self._metadata[0].recorded_start_date if self._metadata else None + return self._first_video_start @property def last_video_end(self) -> Optional[datetime]: - return self._metadata[-1].end if self._metadata else None + return self._last_video_end class TracksMetadata(TrackListObserver): From 9816a4fce022e3021408a0acbf8582f480cdb90c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 16:45:08 +0100 Subject: [PATCH 26/49] Introduce optional fps attribute at video --- OTAnalytics/domain/video.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/domain/video.py b/OTAnalytics/domain/video.py index fb7209787..48bc15aaa 100644 --- a/OTAnalytics/domain/video.py +++ b/OTAnalytics/domain/video.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime, timedelta +from math import floor from os import path from os.path import normcase, splitdrive from pathlib import Path @@ -92,10 +93,11 @@ class SimpleVideo(Video): video_reader: VideoReader path: Path start_date: Optional[datetime] + _fps: Optional[int] = None @property def fps(self) -> float: - return self.video_reader.get_fps(self.path) + return self._fps if self._fps else self.video_reader.get_fps(self.path) def __post_init__(self) -> None: self.check_path_exists() @@ -124,7 +126,8 @@ def get_frame_number_for(self, date: datetime) -> int: time_in_video = date - self.start_date if time_in_video < timedelta(0): return 0 - return self.video_reader.get_frame_number_for(self.path, time_in_video) + + return floor(self.fps * time_in_video.total_seconds()) def to_dict( self, From e82a92fa1d4516da3050d9bd3d52a5b6a65eaa91 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 16:46:39 +0100 Subject: [PATCH 27/49] Reuse filter to plot used point of bounding box --- .../track_visualization/track_viz.py | 78 +++++++++++++++---- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 42abeb61d..e0e28e6de 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -1,6 +1,5 @@ import random from abc import ABC, abstractmethod -from datetime import datetime from typing import Iterable, Optional import numpy @@ -14,7 +13,11 @@ from pandas import DataFrame from PIL import Image -from OTAnalytics.application.plotting import DynamicLayersPlotter, EntityPlotterFactory +from OTAnalytics.application.plotting import ( + DynamicLayersPlotter, + EntityPlotterFactory, + GetCurrentFrame, +) from OTAnalytics.application.state import ( FlowState, Plotter, @@ -678,6 +681,25 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: ) +class FilterByFrame(PandasDataFrameProvider): + def __init__( + self, + data_provider: PandasDataFrameProvider, + current_frame: GetCurrentFrame, + ) -> None: + self._data_provider = data_provider + self._current_frame = current_frame + + def get_data(self) -> DataFrame: + track_df = self._data_provider.get_data() + if track_df.empty: + return track_df + current_frame = self._current_frame.get_frame_number() + FRAME_OFFSET + current_second = self._current_frame.get_second() + timed_df = track_df[track_df[track.SECONDS] == current_second] + return timed_df[timed_df[track.FRAME] == current_frame] + + class TrackBoundingBoxPlotter(MatplotlibPlotterImplementation): """Plot bounding boxes of detections.""" @@ -707,11 +729,7 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: alpha (float): transparency of the lines axes (Axes): axes to plot on """ - current_frame = self.__current_frame() + FRAME_OFFSET - current_second = self.__current_second(track_df) - timed_df = track_df[track_df[track.SECONDS] == current_second] - boxes_frame = timed_df[timed_df[track.FRAME] == current_frame] - for index, row in boxes_frame.iterrows(): + for index, row in track_df.iterrows(): x = row[X] y = row[Y] width = row[W] @@ -730,16 +748,44 @@ def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: ) ) - def __current_frame(self) -> int: - if end_date := self._track_view_state.filter_element.get().date_range.end_date: - video = self._track_view_state.selected_videos.get()[0] - return video.get_frame_number_for(end_date) - return 0 - def __current_second(self, data: DataFrame) -> Optional[datetime]: - if end_date := self._track_view_state.filter_element.get().date_range.end_date: - return end_date.replace(microsecond=0) - return data[track.SECONDS].min() +class TrackPointPlotter(MatplotlibPlotterImplementation): + """Plot point of bounding boxes of detections.""" + + def __init__( + self, + data_provider: PandasDataFrameProvider, + color_palette_provider: ColorPaletteProvider, + track_view_state: TrackViewState, + alpha: float = 0.5, + ) -> None: + self._data_provider = data_provider + self._color_palette_provider = color_palette_provider + self._track_view_state = track_view_state + self._alpha = alpha + + def plot(self, axes: Axes) -> None: + data = self._data_provider.get_data() + if not data.empty: + self._plot_dataframe(data, axes) + + def _plot_dataframe(self, track_df: DataFrame, axes: Axes) -> None: + """ + Plot given tracks on the given axes with the given transparency (alpha) + + Args: + track_df (DataFrame): tracks to plot + axes (Axes): axes to plot on + """ + for index, row in track_df.iterrows(): + classification = row[track.TRACK_CLASSIFICATION] + color = self._color_palette_provider.get()[classification] + axes.plot( + row[X], + row[Y], + marker="x", + color=color, + ) class MatplotlibTrackPlotter(TrackPlotter): From a0f49f1b6520fc7c45d574110986f38568ee6e34 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 18 Dec 2023 16:51:49 +0100 Subject: [PATCH 28/49] Show track points of bounding boxes of current frame --- OTAnalytics/plugin_ui/main_application.py | 5 ++- .../plugin_ui/visualization/visualization.py | 32 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_ui/main_application.py b/OTAnalytics/plugin_ui/main_application.py index 33baad70c..47a95571c 100644 --- a/OTAnalytics/plugin_ui/main_application.py +++ b/OTAnalytics/plugin_ui/main_application.py @@ -233,10 +233,12 @@ def start_gui(self) -> None: section_repository.register_section_changed_observer( clear_all_intersections.on_section_changed ) + videos_metadata = VideosMetadata() layers = self._create_layers( datastore, intersection_repository, track_view_state, + videos_metadata, flow_state, section_state, pulling_progressbar_builder, @@ -258,7 +260,6 @@ def start_gui(self) -> None: tracks_metadata._classifications.register( observer=color_palette_provider.update ) - videos_metadata = VideosMetadata() action_state = self._create_action_state() filter_element_settings_restorer = ( self._create_filter_element_setting_restorer() @@ -605,6 +606,7 @@ def _create_layers( datastore: Datastore, intersection_repository: IntersectionRepository, track_view_state: TrackViewState, + videos_metadata: VideosMetadata, flow_state: FlowState, section_state: SectionState, pulling_progressbar_builder: ProgressbarBuilder, @@ -615,6 +617,7 @@ def _create_layers( datastore, intersection_repository, track_view_state, + videos_metadata, section_state, color_palette_provider, pulling_progressbar_builder, diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index d0febb324..4b53a7a2c 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -6,6 +6,7 @@ from OTAnalytics.application.datastore import Datastore from OTAnalytics.application.plotting import ( CachedPlotter, + GetCurrentFrame, PlottingLayer, TrackBackgroundPlotter, VisualizationTimeProvider, @@ -15,6 +16,7 @@ Plotter, SectionState, TrackViewState, + VideosMetadata, ) from OTAnalytics.application.use_cases.highlight_intersections import ( IntersectionRepository, @@ -42,6 +44,7 @@ ColorPaletteProvider, EventToFlowResolver, FilterByClassification, + FilterByFrame, FilterById, FilterByOccurrence, FlowLayerPlotter, @@ -52,6 +55,7 @@ SectionLayerPlotter, TrackBoundingBoxPlotter, TrackGeometryPlotter, + TrackPointPlotter, TrackStartEndPointPlotter, ) @@ -83,6 +87,7 @@ def __init__( datastore: Datastore, intersection_repository: IntersectionRepository, track_view_state: TrackViewState, + videos_metadata: VideosMetadata, section_state: SectionState, color_palette_provider: ColorPaletteProvider, pulling_progressbar_builder: ProgressbarBuilder, @@ -97,6 +102,9 @@ def __init__( self._flow_repository = datastore._flow_repository self._intersection_repository = intersection_repository self._event_repository = datastore._event_repository + self._get_current_frame = GetCurrentFrame( + track_view_state, videos_metadata, datastore._video_repository + ) self._visualization_time_provider: VisualizationTimeProvider = ( FilterEndDateProvider(track_view_state) ) @@ -136,6 +144,7 @@ def build( ) track_bounding_box_plotter = self._create_track_bounding_box_plotter() + track_point_plotter = self._create_track_point_plotter() layer_definitions = [ ("Background", background_image_plotter, True), @@ -180,6 +189,11 @@ def build( track_bounding_box_plotter, False, ), + ( + "Show track point of bounding boxes of current frame", + track_point_plotter, + False, + ), ] return [ @@ -652,7 +666,23 @@ def _create_track_bounding_box_plotter( ) -> Plotter: track_plotter = MatplotlibTrackPlotter( TrackBoundingBoxPlotter( - self._get_data_provider_class_filter(), + FilterByFrame( + self._get_data_provider_class_filter(), self._get_current_frame + ), + self._color_palette_provider, + self._track_view_state, + alpha=ALPHA_BOUNDING_BOX, + ), + ) + return PlotterPrototype(self._track_view_state, track_plotter) + + def _create_track_point_plotter(self) -> Plotter: + track_plotter = MatplotlibTrackPlotter( + TrackPointPlotter( + FilterByFrame( + self._get_data_provider_all_filters_with_offset(), + self._get_current_frame, + ), self._color_palette_provider, self._track_view_state, alpha=ALPHA_BOUNDING_BOX, From db5cd6bf3eb3eafc0c2e00957806659097cb82b0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Dec 2023 09:07:40 +0100 Subject: [PATCH 29/49] Use use case to get the current frame information --- OTAnalytics/application/datastore.py | 6 + OTAnalytics/application/plotting.py | 16 ++- OTAnalytics/application/state.py | 28 ++++- OTAnalytics/domain/video.py | 11 +- OTAnalytics/plugin_parser/otvision_parser.py | 4 + .../plugin_ui/visualization/visualization.py | 4 +- .../OTAnalytics/application/test_datastore.py | 29 ++--- tests/OTAnalytics/application/test_state.py | 108 ++++++++++++------ .../use_cases/test_load_track_files.py | 4 +- tests/OTAnalytics/domain/test_video.py | 10 +- tests/test_plotting.py | 47 ++++++++ 11 files changed, 195 insertions(+), 72 deletions(-) create mode 100644 tests/test_plotting.py diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index 18c157df3..22364cfa3 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -61,6 +61,12 @@ def duration(self) -> timedelta: return self.expected_duration return timedelta(seconds=self.number_of_frames / self.recorded_fps) + @property + def fps(self) -> float: + if self.actual_fps: + return self.actual_fps + return self.recorded_fps + @dataclass(frozen=True) class TrackParseResult: diff --git a/OTAnalytics/application/plotting.py b/OTAnalytics/application/plotting.py index 5eff40587..858407e32 100644 --- a/OTAnalytics/application/plotting.py +++ b/OTAnalytics/application/plotting.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod -from datetime import datetime +from datetime import datetime, timedelta +from math import floor from typing import Any, Callable, Generic, Iterable, Optional, Sequence, TypeVar from OTAnalytics.application.state import ( @@ -10,7 +11,7 @@ VideosMetadata, ) from OTAnalytics.domain.track import TrackImage -from OTAnalytics.domain.video import Video, VideoRepository +from OTAnalytics.domain.video import Video class Layer: @@ -264,16 +265,19 @@ def __init__( self, state: TrackViewState, videos_metadata: VideosMetadata, - video_repository: VideoRepository, ) -> None: self._state = state self._videos_metadata = videos_metadata - self._video_repository = video_repository def get_frame_number(self) -> int: if end_date := self._state.filter_element.get().date_range.end_date: - video = self._state.selected_videos.get()[0] - return video.get_frame_number_for(end_date) + if metadata := self._videos_metadata.get_metadata_for(end_date): + time_in_video = end_date - metadata.start + if time_in_video < timedelta(0): + return 0 + if time_in_video > metadata.duration: + return metadata.number_of_frames + return floor(metadata.fps * time_in_video.total_seconds()) return 0 def get_second(self) -> Optional[datetime]: diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index ef88faa4f..abb7e6d3e 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -1,3 +1,4 @@ +import bisect from abc import ABC, abstractmethod from datetime import datetime from typing import Callable, Generic, Iterable, Optional @@ -408,12 +409,20 @@ def _update_image(self) -> None: class VideosMetadata: def __init__(self) -> None: - self._metadata: dict[str, VideoMetadata] = {} + self._metadata: dict[datetime, VideoMetadata] = {} self._first_video_start: Optional[datetime] = None self._last_video_end: Optional[datetime] = None def update(self, metadata: VideoMetadata) -> None: - self._metadata[metadata.path] = metadata + """ + Update the stored metadata. + """ + if metadata.start in self._metadata.keys(): + raise ValueError( + f"metadata with start date {metadata.start} already exists." + ) + self._metadata[metadata.start] = metadata + self._metadata = dict(sorted(self._metadata.items())) self._update_start_end_by(metadata) def _update_start_end_by(self, metadata: VideoMetadata) -> None: @@ -422,6 +431,21 @@ def _update_start_end_by(self, metadata: VideoMetadata) -> None: if (not self._last_video_end) or metadata.end > self._last_video_end: self._last_video_end = metadata.end + def get_metadata_for(self, current: datetime) -> Optional[VideoMetadata]: + """ + Find the metadata for the given datetime. If the datetime matches exactly a + start time of a video, the corresponding VideoMetadata is returned. Otherwise, + the metadata of the video containing the datetime will be returned. + """ + if current in self._metadata: + return self._metadata[current] + keys = list(self._metadata.keys()) + key = bisect.bisect_left(keys, current) - 1 + metadata = self._metadata[keys[key]] + if metadata.start <= current <= metadata.end: + return metadata + return None + @property def first_video_start(self) -> Optional[datetime]: return self._first_video_start diff --git a/OTAnalytics/domain/video.py b/OTAnalytics/domain/video.py index 48bc15aaa..72bf030d6 100644 --- a/OTAnalytics/domain/video.py +++ b/OTAnalytics/domain/video.py @@ -35,6 +35,11 @@ def get_frame_number_for(self, video_path: Path, date: timedelta) -> int: class Video(ABC): + @property + @abstractmethod + def start_date(self) -> Optional[datetime]: + raise NotImplementedError + @property @abstractmethod def fps(self) -> float: @@ -92,9 +97,13 @@ class SimpleVideo(Video): video_reader: VideoReader path: Path - start_date: Optional[datetime] + _start_date: Optional[datetime] _fps: Optional[int] = None + @property + def start_date(self) -> Optional[datetime]: + return self._start_date + @property def fps(self) -> float: return self._fps if self._fps else self.video_reader.get_fps(self.path) diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index 82a0124af..3047fde8d 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -824,6 +824,10 @@ class CachedVideo(Video): other: Video cache: dict[int, TrackImage] = field(default_factory=dict) + @property + def start_date(self) -> Optional[datetime]: + return self.other.start_date + @property def fps(self) -> float: return self.other.fps diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 4b53a7a2c..9ff07da72 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -102,9 +102,7 @@ def __init__( self._flow_repository = datastore._flow_repository self._intersection_repository = intersection_repository self._event_repository = datastore._event_repository - self._get_current_frame = GetCurrentFrame( - track_view_state, videos_metadata, datastore._video_repository - ) + self._get_current_frame = GetCurrentFrame(track_view_state, videos_metadata) self._visualization_time_provider: VisualizationTimeProvider = ( FilterEndDateProvider(track_view_state) ) diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index b6dfb11c1..8b536138c 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -71,29 +71,34 @@ def get_frame_number_for(self, video_path: Path, date: timedelta) -> int: class TestVideoMetadata: def test_fully_specified_metadata(self) -> None: + recorded_fps = 20.0 + actual_fps = 20.0 metadata = VideoMetadata( path="video_path_1.mp4", recorded_start_date=FIRST_START_DATE, expected_duration=timedelta(seconds=3), - recorded_fps=20.0, - actual_fps=20.0, + recorded_fps=recorded_fps, + actual_fps=actual_fps, number_of_frames=60, ) assert metadata.start == FIRST_START_DATE assert metadata.end == FIRST_START_DATE + timedelta(seconds=3) + assert metadata.fps == actual_fps def test_partially_specified_metadata(self) -> None: + recorded_fps = 20.0 metadata = VideoMetadata( path="video_path_1.mp4", recorded_start_date=FIRST_START_DATE, expected_duration=None, - recorded_fps=20.0, + recorded_fps=recorded_fps, actual_fps=None, number_of_frames=60, ) assert metadata.start == FIRST_START_DATE expected_video_end = FIRST_START_DATE + timedelta(seconds=3) assert metadata.end == expected_video_end + assert metadata.fps == recorded_fps class TestSimpleVideo: @@ -101,27 +106,15 @@ class TestSimpleVideo: def test_raise_error_if_file_not_exists(self) -> None: with pytest.raises(ValueError): - SimpleVideo( - video_reader=self.video_reader, - path=Path("foo/bar.mp4"), - start_date=FIRST_START_DATE, - ) + SimpleVideo(self.video_reader, Path("foo/bar.mp4"), FIRST_START_DATE) def test_init_with_valid_args(self, cyclist_video: Path) -> None: - video = SimpleVideo( - video_reader=self.video_reader, - path=cyclist_video, - start_date=FIRST_START_DATE, - ) + video = SimpleVideo(self.video_reader, cyclist_video, FIRST_START_DATE) assert video.path == cyclist_video assert video.video_reader == self.video_reader def test_get_frame_return_correct_image(self, cyclist_video: Path) -> None: - video = SimpleVideo( - video_reader=self.video_reader, - path=cyclist_video, - start_date=FIRST_START_DATE, - ) + video = SimpleVideo(self.video_reader, cyclist_video, FIRST_START_DATE) assert video.get_frame(0).as_image() == Image.fromarray( array([[1, 0], [0, 1]], int32) ) diff --git a/tests/OTAnalytics/application/test_state.py b/tests/OTAnalytics/application/test_state.py index 40ab5a341..58a8fa85e 100644 --- a/tests/OTAnalytics/application/test_state.py +++ b/tests/OTAnalytics/application/test_state.py @@ -216,40 +216,43 @@ def test_update_image(self) -> None: plotter.plot.assert_called_once() -class TestVideosMetadata: - @pytest.fixture - def first_full_metadata(self) -> VideoMetadata: - return VideoMetadata( - path="video_path_1.mp4", - recorded_start_date=FIRST_START_DATE, - expected_duration=timedelta(seconds=3), - recorded_fps=20.0, - actual_fps=20.0, - number_of_frames=60, - ) - - @pytest.fixture - def second_full_metadata(self) -> VideoMetadata: - return VideoMetadata( - path="video_path_2.mp4", - recorded_start_date=SECOND_START_DATE, - expected_duration=timedelta(seconds=3), - recorded_fps=20.0, - actual_fps=20.0, - number_of_frames=60, - ) +@pytest.fixture +def first_full_metadata() -> VideoMetadata: + return VideoMetadata( + path="video_path_1.mp4", + recorded_start_date=FIRST_START_DATE, + expected_duration=timedelta(seconds=3), + recorded_fps=20.0, + actual_fps=20.0, + number_of_frames=60, + ) + + +@pytest.fixture +def second_full_metadata() -> VideoMetadata: + return VideoMetadata( + path="video_path_2.mp4", + recorded_start_date=SECOND_START_DATE, + expected_duration=timedelta(seconds=3), + recorded_fps=20.0, + actual_fps=20.0, + number_of_frames=60, + ) + + +@pytest.fixture +def first_partial_metadata() -> VideoMetadata: + return VideoMetadata( + path="video_path_1.mp4", + recorded_start_date=FIRST_START_DATE, + expected_duration=None, + recorded_fps=20.0, + actual_fps=None, + number_of_frames=60, + ) - @pytest.fixture - def first_partial_metadata(self) -> VideoMetadata: - return VideoMetadata( - path="video_path_1.mp4", - recorded_start_date=FIRST_START_DATE, - expected_duration=None, - recorded_fps=20.0, - actual_fps=None, - number_of_frames=60, - ) +class TestVideosMetadata: def test_nothing_updated(self) -> None: videos_metadata = VideosMetadata() @@ -312,6 +315,47 @@ def test_ensure_order( seconds=3 ) + def test_add_metadata_with_same_start_date_fails( + self, first_full_metadata: VideoMetadata, first_partial_metadata: VideoMetadata + ) -> None: + videos_metadata = VideosMetadata() + + videos_metadata.update(first_full_metadata) + with pytest.raises(ValueError): + videos_metadata.update(first_partial_metadata) + + def test_get_metadata_for_date( + self, + first_full_metadata: VideoMetadata, + second_full_metadata: VideoMetadata, + ) -> None: + metadata = VideosMetadata() + + metadata.update(first_full_metadata) + metadata.update(second_full_metadata) + + exact_result = metadata.get_metadata_for(first_full_metadata.start) + floored_result = metadata.get_metadata_for( + first_full_metadata.start + timedelta(seconds=1) + ) + + assert exact_result == first_full_metadata + assert floored_result == first_full_metadata + + def test_get_metadata_for_too_late_date( + self, + first_full_metadata: VideoMetadata, + ) -> None: + metadata = VideosMetadata() + + metadata.update(first_full_metadata) + + result = metadata.get_metadata_for( + first_full_metadata.start + timedelta(seconds=4) + ) + + assert result is None + class TestTracksMetadata: @pytest.fixture diff --git a/tests/OTAnalytics/application/use_cases/test_load_track_files.py b/tests/OTAnalytics/application/use_cases/test_load_track_files.py index 9eaaa8702..a15808c7c 100644 --- a/tests/OTAnalytics/application/use_cases/test_load_track_files.py +++ b/tests/OTAnalytics/application/use_cases/test_load_track_files.py @@ -37,9 +37,7 @@ def test_load(self) -> None: some_track = Mock() some_track_id = TrackId("1") some_track.id = some_track_id - some_video = SimpleVideo( - video_reader=Mock(), path=Path(""), start_date=START_DATE - ) + some_video = SimpleVideo(Mock(), Path(""), START_DATE) detection_metadata = Mock() parse_result = Mock() parse_result.tracks = [some_track] diff --git a/tests/OTAnalytics/domain/test_video.py b/tests/OTAnalytics/domain/test_video.py index bc449df0c..68cfb0b72 100644 --- a/tests/OTAnalytics/domain/test_video.py +++ b/tests/OTAnalytics/domain/test_video.py @@ -33,9 +33,7 @@ def test_resolve_relative_paths( config_path.parent.mkdir(parents=True) video_path.touch() config_path.touch() - video = SimpleVideo( - path=video_path, video_reader=video_reader, start_date=START_DATE - ) + video = SimpleVideo(video_reader, video_path, START_DATE) result = video.to_dict(config_path) @@ -46,9 +44,7 @@ def test_resolve_relative_paths_on_different_drives( ) -> None: video_path = Mock(spec=Path) config_path = Mock(spec=Path) - video = SimpleVideo( - path=video_path, video_reader=video_reader, start_date=START_DATE - ) + video = SimpleVideo(video_reader, video_path, START_DATE) with patch( "OTAnalytics.domain.video.splitdrive", @@ -77,7 +73,7 @@ def test_remove(self, video_reader: VideoReader, test_data_tmp_dir: Path) -> Non observer = Mock(spec=VideoListObserver) path = test_data_tmp_dir / "dummy.mp4" path.touch() - video = SimpleVideo(video_reader, path, start_date=START_DATE) + video = SimpleVideo(video_reader, path, START_DATE) repository = VideoRepository() repository.register_videos_observer(observer) diff --git a/tests/test_plotting.py b/tests/test_plotting.py new file mode 100644 index 000000000..2c46b6020 --- /dev/null +++ b/tests/test_plotting.py @@ -0,0 +1,47 @@ +from datetime import datetime, timedelta +from unittest.mock import Mock + +import pytest +from application.datastore import VideoMetadata +from domain.date import DateRange +from domain.filter import FilterElement + +from OTAnalytics.application.plotting import GetCurrentFrame +from OTAnalytics.application.state import TrackViewState, VideosMetadata + + +class TestGetCurrentFrame: + @pytest.mark.parametrize( + "filter_end_date, expected_frame_number", + [ + (datetime(2023, 1, 1, 0, 1), 0), + (datetime(2023, 1, 1, 0, 1, 1), 20), + (datetime(2023, 1, 1, 0, 1, 3), 60), + (datetime(2023, 1, 1, 0, 1, 4), 60), + ], + ) + def test_get_frame_number( + self, + filter_end_date: datetime, + expected_frame_number: int, + ) -> None: + video_start_date = datetime(2023, 1, 1, 0, 1) + mocked_filter_element = FilterElement( + DateRange(start_date=None, end_date=filter_end_date), classifications={} + ) + state = TrackViewState() + state.filter_element.set(mocked_filter_element) + metadata = Mock(spec=VideoMetadata) + metadata.start = video_start_date + metadata.duration = timedelta(seconds=3) + metadata.fps = 20 + metadata.number_of_frames = 60 + videos_metadata = Mock(spec=VideosMetadata) + videos_metadata.get_metadata_for.return_value = metadata + use_case = GetCurrentFrame(state, videos_metadata) + + frame_number = use_case.get_frame_number() + + assert frame_number == expected_frame_number + + videos_metadata.get_metadata_for.assert_called_with(filter_end_date) From bf35f358046981bdc32a027227e6e215a7ec5501 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Dec 2023 10:25:13 +0100 Subject: [PATCH 30/49] Use correct fps to calculate step size --- OTAnalytics/application/application.py | 60 ++++++++++++++------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/OTAnalytics/application/application.py b/OTAnalytics/application/application.py index 4c26e0825..8120de11d 100644 --- a/OTAnalytics/application/application.py +++ b/OTAnalytics/application/application.py @@ -450,40 +450,44 @@ def get_current_track_offset(self) -> Optional[RelativeOffsetCoordinate]: return self.track_view_state.track_offset.get() def next_frame(self) -> None: - if videos := self.track_view_state.selected_videos.get(): - fps = videos[0].fps - filter_element = self.track_view_state.filter_element.get() - skip_time = self.track_view_state.skip_time.get() - subseconds = min(skip_time.frames, fps) / fps - current_skip = timedelta(seconds=skip_time.seconds) + timedelta( - seconds=subseconds - ) + if filter_element := self.track_view_state.filter_element.get(): current_date_range = filter_element.date_range if current_date_range.start_date and current_date_range.end_date: - next_start = current_date_range.start_date + current_skip - next_end = current_date_range.end_date + current_skip - next_date_range = DateRange(next_start, next_end) - self.track_view_state.filter_element.set( - filter_element.derive_date(next_date_range) - ) + if metadata := self._videos_metadata.get_metadata_for( + current_date_range.end_date + ): + fps = metadata.fps + skip_time = self.track_view_state.skip_time.get() + subseconds = min(skip_time.frames, fps) / fps + current_skip = timedelta(seconds=skip_time.seconds) + timedelta( + seconds=subseconds + ) + next_start = current_date_range.start_date + current_skip + next_end = current_date_range.end_date + current_skip + next_date_range = DateRange(next_start, next_end) + self.track_view_state.filter_element.set( + filter_element.derive_date(next_date_range) + ) def previous_frame(self) -> None: - if videos := self.track_view_state.selected_videos.get(): - fps = videos[0].fps - filter_element = self.track_view_state.filter_element.get() - skip_time = self.track_view_state.skip_time.get() - subseconds = min(skip_time.frames, fps) / fps - current_skip = timedelta(seconds=skip_time.seconds) + timedelta( - seconds=subseconds - ) + if filter_element := self.track_view_state.filter_element.get(): current_date_range = filter_element.date_range if current_date_range.start_date and current_date_range.end_date: - next_start = current_date_range.start_date - current_skip - next_end = current_date_range.end_date - current_skip - next_date_range = DateRange(next_start, next_end) - self.track_view_state.filter_element.set( - filter_element.derive_date(next_date_range) - ) + if metadata := self._videos_metadata.get_metadata_for( + current_date_range.end_date + ): + fps = metadata.fps + skip_time = self.track_view_state.skip_time.get() + subseconds = min(skip_time.frames, fps) / fps + current_skip = timedelta(seconds=skip_time.seconds) + timedelta( + seconds=subseconds + ) + next_start = current_date_range.start_date - current_skip + next_end = current_date_range.end_date - current_skip + next_date_range = DateRange(next_start, next_end) + self.track_view_state.filter_element.set( + filter_element.derive_date(next_date_range) + ) def update_date_range_tracks_filter(self, date_range: DateRange) -> None: """Update the date range of the track filter. From 37416996a40f9fef275063830e5e19fe0e0e480c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Dec 2023 10:25:37 +0100 Subject: [PATCH 31/49] Clean up filter --- .../plugin_prototypes/track_visualization/track_viz.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index e0e28e6de..b60eb1a1c 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -695,9 +695,7 @@ def get_data(self) -> DataFrame: if track_df.empty: return track_df current_frame = self._current_frame.get_frame_number() + FRAME_OFFSET - current_second = self._current_frame.get_second() - timed_df = track_df[track_df[track.SECONDS] == current_second] - return timed_df[timed_df[track.FRAME] == current_frame] + return track_df[track_df[track.FRAME] == current_frame] class TrackBoundingBoxPlotter(MatplotlibPlotterImplementation): From e0301a77c394d14f3448990a95a0b5f340512768 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Dec 2023 10:25:47 +0100 Subject: [PATCH 32/49] Fix data providers --- OTAnalytics/plugin_ui/visualization/visualization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 9ff07da72..692286d6b 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -678,7 +678,7 @@ def _create_track_point_plotter(self) -> Plotter: track_plotter = MatplotlibTrackPlotter( TrackPointPlotter( FilterByFrame( - self._get_data_provider_all_filters_with_offset(), + self._get_data_provider_class_filter_with_offset(), self._get_current_frame, ), self._color_palette_provider, From ec78e7d599721d48719113e02e7b5042a95e2411 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Dec 2023 10:26:08 +0100 Subject: [PATCH 33/49] Use project dir to start tests --- .run/pytest-in-tests.run.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.run/pytest-in-tests.run.xml b/.run/pytest-in-tests.run.xml index 5f90722c2..edc3ff2b2 100644 --- a/.run/pytest-in-tests.run.xml +++ b/.run/pytest-in-tests.run.xml @@ -4,7 +4,7 @@