From 870f5466d22019c76b429a4cb4a4633e7a181fdd Mon Sep 17 00:00:00 2001 From: TomDarmon <36815861+TomDarmon@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:51:55 +0100 Subject: [PATCH] Fea/release semantic versionning (#7) * refacto: rename package name * fix: poetry lock * refacto: adapt makefile+pyproject order+template * fea: release workflow * fix: custom token * fix: poetry in CI * fix: author * fix :poetry core * fix: install cython * fix: require cython * fix: GH token name * fix: doc * fix: install poetry pip * fix: syntax * fix: remove path * fix: add lapx as dependency * fix: official script * fix: remove lock * fix: poetry lock ignore * fix: remove tests * fix: eol --------- Co-authored-by: TomDarmon --- .github/workflows/ci.yaml | 15 +++-- .github/workflows/deploy_docs.yaml | 2 - .github/workflows/release.yaml | 36 +++++++++++ .gitignore | 5 +- Makefile | 44 ++++++++++---- pyproject.toml | 59 +++++++++++++++---- requirements-dev.txt | 12 ---- requirements.txt | 7 --- templates/CHANGELOG.md.j2 | 21 +++++++ {track_reid => trackreid}/args/reid_args.py | 0 .../constants/reid_constants.py | 0 {track_reid => trackreid}/matcher.py | 1 - {track_reid => trackreid}/reid_processor.py | 8 --- {track_reid => trackreid}/tracked_object.py | 3 - .../tracked_object_filter.py | 0 .../tracked_object_metadata.py | 0 {track_reid => trackreid}/utils.py | 2 - 17 files changed, 149 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/release.yaml delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt create mode 100644 templates/CHANGELOG.md.j2 rename {track_reid => trackreid}/args/reid_args.py (100%) rename {track_reid => trackreid}/constants/reid_constants.py (100%) rename {track_reid => trackreid}/matcher.py (99%) rename {track_reid => trackreid}/reid_processor.py (99%) rename {track_reid => trackreid}/tracked_object.py (99%) rename {track_reid => trackreid}/tracked_object_filter.py (100%) rename {track_reid => trackreid}/tracked_object_metadata.py (100%) rename {track_reid => trackreid}/utils.py (99%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 573d30a..0a61283 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,15 +11,18 @@ jobs: python-version: ['3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Install poetry + run: make download-poetry + - name: Install requirements run: | - python -m pip install --upgrade pip - make install_project_requirements - - name: Run Pre commit hook (formatting, linting & tests) - run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure + poetry install + + - name: Run Pre commit hooks + run: make format-code diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml index 5a5e483..3ac59df 100644 --- a/.github/workflows/deploy_docs.yaml +++ b/.github/workflows/deploy_docs.yaml @@ -23,5 +23,3 @@ jobs: run: | mkdocs build mkdocs gh-deploy --force - - \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..595481b --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,36 @@ +# This workflow triggers the CI, updates the version, and uploads the release to GitHub and Google Cloud Storage when a push is made to either the 'main' or 'develop' branch. +# +# Workflow Steps: +# +# 1. Ci is triggered using the CI workflow defined in .github/workflows/ci.yaml +# 2. If it succeeds, the version is updated using Python Semantic Release +# 3. The release is uploaded to GitHub (same step and GitHub action) + +name: CI and Release on main + +on: + push: + branches: + - main + +jobs: + CI: + uses: ./.github/workflows/ci.yaml + + Release: + runs-on: ubuntu-latest + concurrency: Release + needs: CI + permissions: + id-token: write + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Python Semantic Release + uses: python-semantic-release/python-semantic-release@master + with: + github_token: ${{ secrets.GH_TOKEN }} diff --git a/.gitignore b/.gitignore index 2a6bcee..8234a9f 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,7 @@ secrets/* # Data ignore everythin data/detections and data/frames data/detections/* -data/frames/* \ No newline at end of file +data/frames/* + +# poetry +poetry.lock diff --git a/Makefile b/Makefile index 5e0b86e..1385ca7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +PYTHON_VERSION = 3.10 USE_CONDA ?= 1 INSTALL_SCRIPT = install_with_conda.sh ifeq (false,$(USE_CONDA)) @@ -9,21 +10,38 @@ endif help: @grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://' -# help: install - Create a virtual environment and install dependencies +# help: download-poetry - Download poetry +download-poetry: + curl -sSL https://install.python-poetry.org | python3 - + +# help: install - Install python dependencies using poetry .PHONY: install install: - @bash bin/$(INSTALL_SCRIPT) - -# help: install_project_requirements - Install prohect requirements -.PHONY: install_project_requirements -install_project_requirements: - @pip install -r requirements-dev.txt - -# help: install_precommit - Install pre-commit hooks -.PHONY: install_precommit -install_precommit: - @pre-commit install -t pre-commit - @pre-commit install -t pre-push + @poetry env use $(PYTHON_VERSION) + @poetry lock -n + @poetry install -n + @poetry run pre-commit install -t pre-commit -t pre-push + +.PHONY: install-requirements +# help: requirements - Install Python Dependencies +install-requirements: + @poetry install -n + + +.PHONY: install-dev-requirements +# help : install-dev-requirements - Install Python Dependencies for development +install-dev-requirements: + @poetry install -n --with dev + +.PHONY: update-requirements +#help: update-requirements - Update Python Dependencies (requirements.txt and requirements-dev.txt) +update-requirements: + @poetry lock -n + +.PHONY: format-code +#help: format-code - Format/lint all-files using pre-commit hooks (black, flake8, isort, ...) +format-code: + @poetry run pre-commit run -a # help: deploy_docs - Deploy documentation to GitHub Pages .PHONY: deploy_docs diff --git a/pyproject.toml b/pyproject.toml index 3209cb7..1557e0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,39 @@ -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "track-reid" -authors = [ - { name = "tristanpepinartefact", email = "tristan.pepin@artefact.com" }, -] # TODO: Add more authors if collaborators are added +[tool.poetry] +name = "trackreid" +authors = ["tristanpepinartefact "] description = "This Git repository is dedicated to the development of a Python library aimed at correcting the results of tracking algorithms. The primary goal of this library is to reconcile and reassign lost or misidentified IDs, ensuring a consistent and accurate tracking of objects over time. " version = "0.0.1" readme = "README.md" -requires-python = ">=3.8" + + +[tool.poetry.dependencies] +python = ">=3.8, <3.11.0" +pandas = "1.5.3" +numpy = "1.24.2" +llist = "0.7.1" +pydantic = "2.4.2" +torch = ">=1.13.1" +Cython = ">=0.29.23" +bytetracker = { git = "https://github.com/TomDarmon/bytetrack-pip.git", branch = "main" } + +[tool.poetry.group.dev.dependencies] +black = "22.10.0" +ruff = "0.0.272" +isort = "5.12.0" +pre-commit = "3.3.3" +pytest = "7.3.2" +mkdocs = "1.4.3" +mkdocs-material = "9.1.15" +mkdocstrings-python = "1.1.2" +bandit = "1.7.5" +nbstripout = "0.6.1" +ipykernel = "6.24.0" + + +[build-system] +requires = ["poetry-core>=1.0.0", "Cython=^0.29.23"] +build-backend = "poetry.core.masonry.api" + [project.urls] "Homepage" = "https://github.com/artefactory-fr/track-reid" @@ -32,7 +55,7 @@ select = [ "PD", ] # See: https://beta.ruff.rs/docs/rules/ ignore = ["D203", "D213", "ANN101", "ANN102"] -line-length = 100 +line-length = 120 target-version = "py310" exclude = [ ".bzr", @@ -78,3 +101,17 @@ exclude = ''' [tool.isort] profile = "black" + + +[tool.semantic_release] +version_variables = ["deployer/__init__.py:__version__"] +version_toml = ["pyproject.toml:tool.poetry.version"] +branch = "main" +upload_to_pypi = false +upload_to_release = true +build_command = "pip install poetry && poetry build" +commit_message = "chore(release): {version}\n\nAutomatically generated by semantic-release" +tag_format = "{version}" + +[tool.semantic_release.changelog] +exclude_commit_patterns = ['''^chore\(release\).*'''] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 27a83a9..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ --r requirements.txt -black==22.10.0 -ruff==0.0.272 -isort==5.12.0 -pre-commit==3.3.3 -pytest==7.3.2 -mkdocs==1.4.3 -mkdocs-material==9.1.15 -mkdocstrings-python==1.1.2 -bandit==1.7.5 -nbstripout==0.6.1 -ipykernel==6.24.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1ff2cc4..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -pandas==1.5.3 -numpy==1.24.2 -llist==0.7.1 -pydantic==2.4.2 - -# custom bytetrack -git+https://github.com/TomDarmon/bytetrack-pip.git@main#egg=bytetracker diff --git a/templates/CHANGELOG.md.j2 b/templates/CHANGELOG.md.j2 new file mode 100644 index 0000000..04906cb --- /dev/null +++ b/templates/CHANGELOG.md.j2 @@ -0,0 +1,21 @@ +# CHANGELOG +{% if context.history.unreleased | length > 0 %} +{# UNRELEASED #} +## Unreleased +{% for type_, commits in context.history.unreleased | dictsort %} +### {{ type_ | capitalize }} +{% for commit in commits %}{% if type_ != "unknown" %} +* {{ commit.message.rstrip() }} ([`{{ commit.short_hash }}`]({{ commit.hexsha | commit_hash_url }})) +{% else %} +* {{ commit.message.rstrip() }} ([`{{ commit.short_hash }}`]({{ commit.hexsha | commit_hash_url }})) +{% endif %}{% endfor %}{% endfor %}{% endif %} +{% for version, release in context.history.released.items() %} +{# RELEASED #} +## {{ version.as_tag() }} ({{ release.tagged_date.strftime("%Y-%m-%d") }}) +{% for type_, commits in release["elements"] | dictsort %} +### {{ type_ | capitalize }} +{% for commit in commits %}{% if type_ != "unknown" %} +* {{ commit.message.rstrip() }} ([`{{ commit.short_hash }}`]({{ commit.hexsha | commit_hash_url }})) +{% else %} +* {{ commit.message.rstrip() }} ([`{{ commit.short_hash }}`]({{ commit.hexsha | commit_hash_url }})) +{% endif %}{% endfor %}{% endfor %}{% endfor %} diff --git a/track_reid/args/reid_args.py b/trackreid/args/reid_args.py similarity index 100% rename from track_reid/args/reid_args.py rename to trackreid/args/reid_args.py diff --git a/track_reid/constants/reid_constants.py b/trackreid/constants/reid_constants.py similarity index 100% rename from track_reid/constants/reid_constants.py rename to trackreid/constants/reid_constants.py diff --git a/track_reid/matcher.py b/trackreid/matcher.py similarity index 99% rename from track_reid/matcher.py rename to trackreid/matcher.py index cb8b817..18702a4 100644 --- a/track_reid/matcher.py +++ b/trackreid/matcher.py @@ -2,7 +2,6 @@ import numpy as np from scipy.optimize import linear_sum_assignment - from track_reid.tracked_object import TrackedObject diff --git a/track_reid/reid_processor.py b/trackreid/reid_processor.py similarity index 99% rename from track_reid/reid_processor.py rename to trackreid/reid_processor.py index d345e1d..57d18a3 100644 --- a/track_reid/reid_processor.py +++ b/trackreid/reid_processor.py @@ -3,7 +3,6 @@ from typing import Dict, List, Set import numpy as np - from track_reid.constants.reid_constants import reid_constants from track_reid.matcher import Matcher from track_reid.tracked_object import TrackedObject @@ -21,7 +20,6 @@ def __init__( max_frames_to_rematch: int = 100, max_attempt_to_rematch: int = 1, ) -> None: - self.matcher = Matcher(cost_function=cost_function, selection_function=selection_function) self.tracked_filter = TrackedObjectFilter( @@ -52,7 +50,6 @@ def _preprocess(self, tracker_output: np.ndarray): self.all_tracked_objects = self._apply_filtering() def _update_tracked_objects(self, tracker_output: np.ndarray): - self.frame_id = tracker_output[0, 0] for object_id, data_line in zip(tracker_output[:, 1], tracker_output): if object_id not in self.all_tracked_objects: @@ -80,7 +77,6 @@ def _apply_filtering(self): return self.all_tracked_objects def _perform_reid_process(self, tracker_output: np.ndarray): - tracked_ids = filter_objects_by_state( self.all_tracked_objects, states=reid_constants.BYETRACK_OUTPUT, exclusion=True ) @@ -151,13 +147,11 @@ def identify_candidates(tracked_ids: List["TrackedObject"]): @staticmethod def compute_stable_objects(tracked_ids: list, current_tracker_ids: Set[int]): - top_list_correction = get_top_list_correction(tracked_ids) for current_object in current_tracker_ids: tracked_id = tracked_ids[tracked_ids.index(current_object)] if current_object not in top_list_correction: - tracked_ids.remove(tracked_id) new_object, tracked_id = tracked_id.cut(current_object) @@ -175,7 +169,6 @@ def process_matches( candidates: List["TrackedObject"], current_tracker_ids: Set[int], ): - for match in matches: candidate_match, switcher_match = match.popitem() @@ -194,7 +187,6 @@ def drop_switchers( max_frames_to_rematch: int, frame_id: int, ): - switchers_to_drop = set(switchers).intersection(current_tracker_ids) filtered_switchers = switchers.copy() diff --git a/track_reid/tracked_object.py b/trackreid/tracked_object.py similarity index 99% rename from track_reid/tracked_object.py rename to trackreid/tracked_object.py index 527b14c..b2dd1dc 100644 --- a/track_reid/tracked_object.py +++ b/trackreid/tracked_object.py @@ -4,7 +4,6 @@ import numpy as np from llist import sllist - from track_reid.constants.reid_constants import reid_constants from track_reid.tracked_object_metadata import TrackedObjectMetaData from track_reid.utils import split_list_around_value @@ -17,7 +16,6 @@ def __init__( state: int, metadata: Union[np.ndarray, TrackedObjectMetaData], ): - self.state = state if isinstance(object_ids, int): @@ -99,7 +97,6 @@ def __eq__(self, other): return False def cut(self, object_id: int): - if object_id not in self.re_id_chain: raise NameError( f"Trying to cut object {self} with {object_id} that is not in the re-id chain." diff --git a/track_reid/tracked_object_filter.py b/trackreid/tracked_object_filter.py similarity index 100% rename from track_reid/tracked_object_filter.py rename to trackreid/tracked_object_filter.py diff --git a/track_reid/tracked_object_metadata.py b/trackreid/tracked_object_metadata.py similarity index 100% rename from track_reid/tracked_object_metadata.py rename to trackreid/tracked_object_metadata.py diff --git a/track_reid/utils.py b/trackreid/utils.py similarity index 99% rename from track_reid/utils.py rename to trackreid/utils.py index 8060d3e..7c559f3 100644 --- a/track_reid/utils.py +++ b/trackreid/utils.py @@ -4,14 +4,12 @@ def get_top_list_correction(tracked_ids: list): - top_list_correction = [tracked_id.re_id_chain.last.value for tracked_id in tracked_ids] return top_list_correction def split_list_around_value(my_list: sllist, value_to_split: int): - if value_to_split == my_list.last.value: raise NameError("split on the last") before = sllist()