From 6fbc3aaf4207d8b96c2f6568accdcc5dcadf67de Mon Sep 17 00:00:00 2001 From: TomDarmon <36815861+TomDarmon@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:34:02 +0100 Subject: [PATCH] fea: norfair demo nb (#3) * fea: norfair demo nb * fix: data folder + script * fix: improve video generation * fix: script * fix: use .txt instead of pip tools * fix: pre-commit --------- Co-authored-by: TomDarmon --- .gitignore | 7 +- .pre-commit-config.yaml | 6 - Makefile | 7 - bin/download_sample_sequences.sh | 20 ++ data/.gitkeep | 0 lib/bbox/utils.py | 25 +++ lib/norfair_helper/utils.py | 25 +++ lib/norfair_helper/video.py | 37 ++++ lib/sequence.py | 39 ++++ notebooks/norfair_starter_kit.ipynb | 252 +++++++++++++++++++++ pyproject.toml | 2 - requirements.in => requirements-dev.txt | 10 +- requirements.txt | 279 +----------------------- 13 files changed, 407 insertions(+), 302 deletions(-) create mode 100644 bin/download_sample_sequences.sh create mode 100644 data/.gitkeep create mode 100644 lib/bbox/utils.py create mode 100644 lib/norfair_helper/utils.py create mode 100644 lib/norfair_helper/video.py create mode 100644 lib/sequence.py create mode 100644 notebooks/norfair_starter_kit.ipynb rename requirements.in => requirements-dev.txt (66%) diff --git a/.gitignore b/.gitignore index 5c7f9f6..2a6bcee 100644 --- a/.gitignore +++ b/.gitignore @@ -135,4 +135,9 @@ secrets/* !secrets/.gitkeep # Mac OS -.DS_Store \ No newline at end of file +.DS_Store + + +# Data ignore everythin data/detections and data/frames +data/detections/* +data/frames/* \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b58f925..6643f2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,12 +31,6 @@ repos: types: [file] files: (.ipynb)$ language: system - - id: python-bandit-vulnerability-check - name: Security check (bandit) - entry: bandit - types: [python] - args: ["--recursive", "lib/"] - language: system - id: pytest-check name: Tests (pytest) stages: [push] diff --git a/Makefile b/Makefile index 9d352bf..8998195 100644 --- a/Makefile +++ b/Makefile @@ -18,17 +18,10 @@ help: install: @bash bin/$(INSTALL_SCRIPT) -# help: install_pip_tools - Install piptools and setuptools -.PHONY: install_pip_tools -install_pip_tools: - @echo "Installing pip-tools" - @pip install pip-tools==$(PIP_TOOLS_VERSION) setuptools==$(SETUPTOOLS_VERSION) - # help: install_project_requirements - Install prohect requirements .PHONY: install_project_requirements install_project_requirements: install_pip_tools @pip install numpy==${NUMPY_VERSION} - @pip-compile requirements.in @pip install -r requirements.txt # help: install_precommit - Install pre-commit hooks diff --git a/bin/download_sample_sequences.sh b/bin/download_sample_sequences.sh new file mode 100644 index 0000000..d357733 --- /dev/null +++ b/bin/download_sample_sequences.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +N_SEQUENCES=5 + +sequences_detections=$(gsutil ls gs://data-track-reid/detections | head -$N_SEQUENCES) +sequences_frames=$(gsutil ls gs://data-track-reid/frames | head -$N_SEQUENCES) + +# remove first sequence which is the bucket name +sequences_detections=$(echo "$sequences_detections" | tail -n +2) +sequences_frames=$(echo "$sequences_frames" | tail -n +2) + + +# download the sequences to data/detections and data/frames +for sequence in $sequences_detections; do + gsutil -m cp -r $sequence data/detections +done + +for sequence in $sequences_frames; do + gsutil -m cp -r $sequence data/frames +done diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/lib/bbox/utils.py b/lib/bbox/utils.py new file mode 100644 index 0000000..82ee2f7 --- /dev/null +++ b/lib/bbox/utils.py @@ -0,0 +1,25 @@ +import numpy as np + + +def xy_center_to_xyxy(bbox: np.array) -> np.array: + """Convert bounding box from xy_center to xyxy format""" + return np.array( + [ + bbox[0] - bbox[2] / 2, + bbox[1] - bbox[3] / 2, + bbox[0] + bbox[2] / 2, + bbox[1] + bbox[3] / 2, + ] + ) + + +def rescale_bbox(bbox: np.array, original_size: tuple) -> np.array: + """Rescale bounding box from current_size to original_size""" + return np.array( + [ + bbox[0] * original_size[0], + bbox[1] * original_size[1], + bbox[2] * original_size[0], + bbox[3] * original_size[1], + ] + ) diff --git a/lib/norfair_helper/utils.py b/lib/norfair_helper/utils.py new file mode 100644 index 0000000..11bf4af --- /dev/null +++ b/lib/norfair_helper/utils.py @@ -0,0 +1,25 @@ +from typing import List + +import numpy as np +from norfair import Detection + +from lib.bbox.utils import rescale_bbox, xy_center_to_xyxy + + +def yolo_to_norfair_detection( + yolo_detections: np.array, original_img_size: tuple +) -> List[Detection]: + """convert detections_as_xywh to norfair detections""" + norfair_detections: List[Detection] = [] + for detection_output in yolo_detections: + bbox = np.array( + [ + [detection_output[1].item(), detection_output[2].item()], + [detection_output[3].item(), detection_output[4].item()], + ] + ) + bbox = xy_center_to_xyxy(bbox.flatten()).reshape(2, 2) + bbox = rescale_bbox(bbox.flatten(), original_img_size).reshape(2, 2) + scores = np.array([detection_output[5].item(), detection_output[5].item()]) + norfair_detections.append(Detection(points=bbox, scores=scores, label=detection_output[0])) + return norfair_detections diff --git a/lib/norfair_helper/video.py b/lib/norfair_helper/video.py new file mode 100644 index 0000000..3ea8530 --- /dev/null +++ b/lib/norfair_helper/video.py @@ -0,0 +1,37 @@ +import cv2 +import numpy as np +from norfair import Tracker, draw_boxes + +from lib.norfair_helper.utils import yolo_to_norfair_detection +from lib.sequence import Sequence + + +def generate_tracking_video( + sequence: Sequence, tracker: Tracker, frame_size: tuple, output_path: str +) -> str: + """ + Generate a video with the tracking results. + + Args: + sequence: The sequence of frames and detections. + tracker: The tracker to use. + frame_size: The size of the frames. + output_path: The path to save the video to. + + Returns: + The path to the video. + + """ + fourcc = cv2.VideoWriter_fourcc(*"mp4v") # Changed codec to 'mp4v' for compatibility with Mac + out = cv2.VideoWriter(output_path, fourcc, 20.0, frame_size) # Changed file extension to .mp4 + + for frame, detection in sequence: + detections_list = yolo_to_norfair_detection(detection, frame_size) + tracked_objects = tracker.update(detections=detections_list) + frame_detected = draw_boxes( + np.array(frame), tracked_objects, draw_ids=True, color="by_label" + ) + frame_detected = cv2.cvtColor(frame_detected, cv2.COLOR_BGR2RGB) + out.write(frame_detected) + out.release() + return output_path diff --git a/lib/sequence.py b/lib/sequence.py new file mode 100644 index 0000000..97b765a --- /dev/null +++ b/lib/sequence.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass +from typing import List + +import numpy as np +from PIL import Image + + +@dataclass +class Sequence: + """This class represents a sequence of frames and detections and make the data iterable.""" + + frame_paths: List[str] + + def __post_init__(self): + self.detection_paths: List[str] = [ + f.replace("frames", "detections").replace(".jpg", ".txt") for f in self.frame_paths + ] + + def __repr__(self): + return ( + f"Sequence(n_frames={len(self.frame_paths)}, n_detections={len(self.detection_paths)})" + ) + + def __iter__(self): + self.index = 0 + return self + + def __next__(self): + if self.index >= len(self.frame_paths): + raise StopIteration + + frame = Image.open(self.frame_paths[self.index]) + try: + detection = np.loadtxt(self.detection_paths[self.index], dtype="float") + except OSError: # file doesn't exist not detection return empty file + detection = np.array([]) + + self.index += 1 + return frame, detection diff --git a/notebooks/norfair_starter_kit.ipynb b/notebooks/norfair_starter_kit.ipynb new file mode 100644 index 0000000..ff9ff59 --- /dev/null +++ b/notebooks/norfair_starter_kit.ipynb @@ -0,0 +1,252 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Value proposition of norfair" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Norfair is a customizable lightweight Python library for real-time multi-object tracking.\n", + "Using Norfair, you can add tracking capabilities to any detector with just a few lines of code.\n", + "\n", + "It means you won't need a SOTA Tracker you can use a basic Tracker with a Kalmann Filter and add the custom logic you want." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Imports and setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys; sys.path.append('..')\n", + "import os\n", + "\n", + "from norfair import Tracker\n", + "\n", + "from lib.sequence import Sequence\n", + "from lib.norfair_helper.video import generate_tracking_video\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to test this code on your detection and frames you can use the following code if you structure the data as follows:\n", + "\n", + "```\n", + "data/\n", + " ├── detection/\n", + " │ └── sequence_1/\n", + " │ └── detections_1.txt\n", + " └── frames/\n", + " └── sequence_1/\n", + " └── frame_1.jpg\n", + "```\n", + "\n", + "Where the detections.txt file is in the following format scaled between 0 and 1:\n", + "\n", + "```\n", + "class_id x_center y_center width height confidence\n", + "```\n", + "\n", + "If this is not the case, you'll need to adapt this code to your data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DATA_PATH = \"../data\"\n", + "DETECTION_PATH = f\"{DATA_PATH}/detections\"\n", + "FRAME_PATH = f\"{DATA_PATH}/frames\"\n", + "VIDEO_OUTPUT_PATH = \"private\"\n", + "\n", + "SEQUENCES = os.listdir(FRAME_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_sequence_frames(sequence):\n", + " frames = os.listdir(f\"{FRAME_PATH}/{sequence}\")\n", + " frames = [os.path.join(f\"{FRAME_PATH}/{sequence}\", frame) for frame in frames]\n", + " frames.sort()\n", + " return frames\n", + "\n", + "def get_sequence_detections(sequence):\n", + " detections = os.listdir(f\"{DETECTION_PATH}/{sequence}\")\n", + " detections = [os.path.join(f\"{DETECTION_PATH}/{sequence}\", detection) for detection in detections]\n", + " detections.sort()\n", + " return detections\n", + "\n", + "frame_path = get_sequence_frames(SEQUENCES[1])\n", + "test_sequence = Sequence(frame_path)\n", + "test_sequence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_sequence" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Usage of Norfair" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tracker\n", + "\n", + "Norfair tracker object is the customizable object that will track detections.\n", + "Norfair expects a distance function that will serve as a metric to match objects between each detection. You can create your own distance metric or use one of the built-in ones such as euclidian distance, iou or many more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize a tracker with the distance function\n", + "basic_tracker = Tracker(\n", + " distance_function=\"mean_euclidean\",\n", + " distance_threshold=40,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic tracking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "video_path = generate_tracking_video(\n", + " sequence=test_sequence,\n", + " tracker=basic_tracker,\n", + " frame_size=(2560, 1440),\n", + " output_path=os.path.join(VIDEO_OUTPUT_PATH, \"basic_tracking.mp4\"),\n", + ")\n", + "video_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced tracking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def reid_distance_advanced(new_object, unmatched_object):\n", + " return 0 # ALWAYS MATCH" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "advanced_tracker = Tracker(\n", + " distance_function=\"sqeuclidean\",\n", + " distance_threshold=350, # Higher value means objects further away will be matched\n", + " initialization_delay=10, # Wait 15 frames before an object is starts to be tracked\n", + " hit_counter_max=20, # Inertia, higher values means an object will take time to enter in reid phase\n", + " reid_distance_function=reid_distance_advanced, # function to decide on which metric to reid\n", + " reid_distance_threshold=0.5, # If the distance is below 0.5 the object is matched\n", + " reid_hit_counter_max=200, # inertia, higher values means an object will enter reid phase longer\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "video_path = generate_tracking_video(\n", + " sequence=test_sequence,\n", + " tracker=advanced_tracker,\n", + " frame_size=(2560, 1440),\n", + " output_path=os.path.join(VIDEO_OUTPUT_PATH, \"advance_tracking.mp4\"),\n", + ")\n", + "video_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "advanced_tracker.total_object_count" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "track-reid", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index 3e8ad07..97e66fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,6 @@ select = [ "F", "I", "N", - "D", - "ANN", "Q", "RET", "ARG", diff --git a/requirements.in b/requirements-dev.txt similarity index 66% rename from requirements.in rename to requirements-dev.txt index 0b40650..27a83a9 100644 --- a/requirements.in +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -# dev +-r requirements.txt black==22.10.0 ruff==0.0.272 isort==5.12.0 @@ -10,11 +10,3 @@ mkdocstrings-python==1.1.2 bandit==1.7.5 nbstripout==0.6.1 ipykernel==6.24.0 - -# features - -pandas==1.5.3 -numpy==1.24.2 -bytetracker==0.3.2 -llist==0.7.1 -pydantic==2.4.2 diff --git a/requirements.txt b/requirements.txt index 8e50e01..b045da3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,280 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -annotated-types==0.6.0 - # via pydantic -appnope==0.1.3 - # via - # ipykernel - # ipython -asttokens==2.4.1 - # via stack-data -attrs==23.1.0 - # via - # jsonschema - # referencing -bandit==1.7.5 - # via -r requirements.in -black==22.10.0 - # via -r requirements.in +pandas==1.5.3 +numpy==1.24.2 bytetracker==0.3.2 - # via -r requirements.in -certifi==2023.7.22 - # via requests -cfgv==3.4.0 - # via pre-commit -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # black - # mkdocs -colorama==0.4.6 - # via - # griffe - # mkdocs-material -comm==0.2.0 - # via ipykernel -debugpy==1.8.0 - # via ipykernel -decorator==5.1.1 - # via ipython -distlib==0.3.7 - # via virtualenv -exceptiongroup==1.1.3 - # via - # ipython - # pytest -executing==2.0.1 - # via stack-data -fastjsonschema==2.18.1 - # via nbformat -filelock==3.13.1 - # via virtualenv -ghp-import==2.1.0 - # via mkdocs -gitdb==4.0.11 - # via gitpython -gitpython==3.1.40 - # via bandit -griffe==0.36.9 - # via mkdocstrings-python -identify==2.5.31 - # via pre-commit -idna==3.4 - # via requests -iniconfig==2.0.0 - # via pytest -ipykernel==6.24.0 - # via -r requirements.in -ipython==8.17.2 - # via ipykernel -isort==5.12.0 - # via -r requirements.in -jedi==0.19.1 - # via ipython -jinja2==3.1.2 - # via - # mkdocs - # mkdocs-material - # mkdocstrings -jsonschema==4.19.2 - # via nbformat -jsonschema-specifications==2023.7.1 - # via jsonschema -jupyter-client==8.6.0 - # via ipykernel -jupyter-core==5.5.0 - # via - # ipykernel - # jupyter-client - # nbformat -lap==0.4.0 - # via bytetracker llist==0.7.1 - # via -r requirements.in -markdown==3.3.7 - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via - # jinja2 - # mkdocstrings -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -mergedeep==1.3.4 - # via mkdocs -mkdocs==1.4.3 - # via - # -r requirements.in - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings -mkdocs-autorefs==0.5.0 - # via mkdocstrings -mkdocs-material==9.1.15 - # via -r requirements.in -mkdocs-material-extensions==1.3 - # via mkdocs-material -mkdocstrings==0.23.0 - # via mkdocstrings-python -mkdocstrings-python==1.1.2 - # via -r requirements.in -mypy-extensions==1.0.0 - # via black -nbformat==5.9.2 - # via nbstripout -nbstripout==0.6.1 - # via -r requirements.in -nest-asyncio==1.5.8 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -numpy==1.24.2 - # via - # -r requirements.in - # pandas - # scipy -packaging==23.2 - # via - # ipykernel - # mkdocs - # pytest -pandas==1.5.3 - # via -r requirements.in -parso==0.8.3 - # via jedi -pathspec==0.11.2 - # via black -pbr==5.11.1 - # via stevedore -pexpect==4.8.0 - # via ipython -platformdirs==3.11.0 - # via - # black - # jupyter-core - # virtualenv -pluggy==1.3.0 - # via pytest -pre-commit==3.3.3 - # via -r requirements.in -prompt-toolkit==3.0.39 - # via ipython -psutil==5.9.6 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data pydantic==2.4.2 - # via -r requirements.in -pydantic-core==2.10.1 - # via pydantic -pygments==2.16.1 - # via - # ipython - # mkdocs-material - # rich -pymdown-extensions==10.3.1 - # via - # mkdocs-material - # mkdocstrings -pytest==7.3.2 - # via -r requirements.in -python-dateutil==2.8.2 - # via - # ghp-import - # jupyter-client - # pandas -pytz==2023.3.post1 - # via pandas -pyyaml==6.0.1 - # via - # bandit - # mkdocs - # pre-commit - # pymdown-extensions - # pyyaml-env-tag -pyyaml-env-tag==0.1 - # via mkdocs -pyzmq==25.1.1 - # via - # ipykernel - # jupyter-client -referencing==0.30.2 - # via - # jsonschema - # jsonschema-specifications -regex==2023.10.3 - # via mkdocs-material -requests==2.31.0 - # via mkdocs-material -rich==13.6.0 - # via bandit -rpds-py==0.12.0 - # via - # jsonschema - # referencing -ruff==0.0.272 - # via -r requirements.in -scipy==1.9.3 - # via bytetracker -six==1.16.0 - # via - # asttokens - # python-dateutil -smmap==5.0.1 - # via gitdb -stack-data==0.6.3 - # via ipython -stevedore==5.1.0 - # via bandit -tomli==2.0.1 - # via - # black - # pytest -torch==1.13.0 - # via bytetracker -tornado==6.3.3 - # via - # ipykernel - # jupyter-client -traitlets==5.13.0 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.8.0 - # via - # pydantic - # pydantic-core - # torch -urllib3==2.0.7 - # via requests -virtualenv==20.24.6 - # via pre-commit -watchdog==3.0.0 - # via mkdocs -wcwidth==0.2.9 - # via prompt-toolkit - -# The following packages are considered to be unsafe in a requirements file: -# setuptools