From 7ed5859c220936b7082d0bc2a426e89b38163a91 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 11:58:31 +0100 Subject: [PATCH 1/8] update pre-commit workflow Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index 7164f84e..92d9ece4 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -15,23 +15,22 @@ env: jobs: pr_tests: - # runs-on: ${{ matrix.os }} - runs-on: [self-hosted, sdk-runner] + runs-on: ${{ matrix.target[0] }} strategy: matrix: - # os: [ubuntu-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10'] + target: [[ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] fail-fast: true max-parallel: 1 steps: - - - name: Checkout code with caching for Git LFS - uses: nschloe/action-cached-lfs-checkout@v1.2.0 + - name: Checkout repository + uses: actions/checkout@v4 + with: + lfs: true - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.target[1] }} - name: Install package with dev and notebook requirements run: | @@ -48,12 +47,15 @@ jobs: - name: Check imports with isort run: isort . --check + if: ${{ always() }} - name: Flake8 linting run: flake8 . + if: ${{ always() }} - name: Pydocstyle linting run: pydocstyle geti_sdk --count + if: ${{ always() }} - name: Test with pytest run: | From adccfa4887d0939e14a2688cf2ae2a7e7abf7729 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 12:03:41 +0100 Subject: [PATCH 2/8] update black Signed-off-by: Igor Davidyuk --- .pre-commit-config.yaml | 2 +- geti_sdk/benchmarking/benchmarker.py | 8 +++++--- geti_sdk/data_models/configurable_parameter_group.py | 6 +++--- geti_sdk/data_models/job.py | 6 +++--- geti_sdk/geti.py | 8 +++++--- requirements/requirements-dev.txt | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba52069e..91206205 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: # Code formatting - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.1.1 hooks: - id: black diff --git a/geti_sdk/benchmarking/benchmarker.py b/geti_sdk/benchmarking/benchmarker.py index 377b9118..00d88861 100644 --- a/geti_sdk/benchmarking/benchmarker.py +++ b/geti_sdk/benchmarking/benchmarker.py @@ -923,9 +923,11 @@ def compare_predictions( "run_name": f"Deployment {deployment_index}", "model_1": deployment.models[0].name, "model_1_score": model_scores[0], - "fps": None - if throughput_benchmark_results is None - else throughput_benchmark_results[deployment_index]["fps"], + "fps": ( + None + if throughput_benchmark_results is None + else throughput_benchmark_results[deployment_index]["fps"] + ), } if not self._is_single_task: model_info.update( diff --git a/geti_sdk/data_models/configurable_parameter_group.py b/geti_sdk/data_models/configurable_parameter_group.py index fb0ccf99..14d9f237 100644 --- a/geti_sdk/data_models/configurable_parameter_group.py +++ b/geti_sdk/data_models/configurable_parameter_group.py @@ -145,9 +145,9 @@ def from_dict(cls, input_dict: Dict[str, Any]) -> "ParameterGroup": input_dict """ input_copy = copy.deepcopy(input_dict) - parameter_dicts: List[ - Union[Dict[str, Any], ConfigurableParameter] - ] = input_copy.pop("parameters", []) + parameter_dicts: List[Union[Dict[str, Any], ConfigurableParameter]] = ( + input_copy.pop("parameters", []) + ) group_dicts: List[Union[Dict[str, Any], ParameterGroup]] = input_copy.pop( "groups", [] ) diff --git a/geti_sdk/data_models/job.py b/geti_sdk/data_models/job.py index 0fe6c9d9..9ae83801 100644 --- a/geti_sdk/data_models/job.py +++ b/geti_sdk/data_models/job.py @@ -162,9 +162,9 @@ class JobMetadata: scores: Optional[List[ScoreMetadata]] = None trained_model: Optional[ModelMetadata] = None # Added in Geti v1.7 warnings: Optional[List[dict]] = None # Added in Geti v1.13 for dataset import jobs - supported_project_types: Optional[ - List[dict] - ] = None # Added in Geti v1.13 for dataset import jobs + supported_project_types: Optional[List[dict]] = ( + None # Added in Geti v1.13 for dataset import jobs + ) project_id: Optional[str] = None # Added in Geti v1.13 for dataset import jobs diff --git a/geti_sdk/geti.py b/geti_sdk/geti.py index 69dbdb0d..8869c085 100644 --- a/geti_sdk/geti.py +++ b/geti_sdk/geti.py @@ -765,9 +765,11 @@ def create_task_chain_project_from_dataset( regarding the uploaded project """ labels_per_task = [ - entry.get_all_label_names() - if isinstance(entry, AnnotationReader) - else entry + ( + entry.get_all_label_names() + if isinstance(entry, AnnotationReader) + else entry + ) for entry in label_source_per_task ] annotation_readers_per_task = [ diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 1ec0b369..df6dee85 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -7,7 +7,7 @@ pytest-env==1.0.* pytest-html==3.2.* flake8==6.0.* pydocstyle>=6.1 -black>=22.6 +black>=24.1 isort>=5.12 pre-commit>=3.5 nbqa>=1.7.0 From 7bd4c2e0d96db10ef2e26ed08207073cbb0a0888 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 12:05:01 +0100 Subject: [PATCH 3/8] focus on the new platform Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index 92d9ece4..ccd3e813 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.target[0] }} strategy: matrix: - target: [[ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] + target: [[ubuntu-latest, '3.8'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] # [ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], fail-fast: true max-parallel: 1 steps: From 8f87a5c4b0c04be0901219647714262491692116 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 12:33:50 +0100 Subject: [PATCH 4/8] try to solve mkdir problem Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index ccd3e813..3e25e22d 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -20,7 +20,7 @@ jobs: matrix: target: [[ubuntu-latest, '3.8'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] # [ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], fail-fast: true - max-parallel: 1 + max-parallel: 5 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -40,7 +40,7 @@ jobs: - name: Create report directory # Creates the temporary directory that is used to store the test report run: | - mkdir $REPORT_DIRECTORY + mkdir -p $REPORT_DIRECTORY - name: Check formatting with black run: black . --check From 63ece2bf7db41ac5cf2069c29da2ffd921067f54 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 13:56:33 +0100 Subject: [PATCH 5/8] solve encoding issue Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index 3e25e22d..99622700 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -12,6 +12,7 @@ on: env: PREMERGE_TEST_REPORT: pre_merge_test_report.html REPORT_DIRECTORY: reports + PYTHONUTF8: '1' jobs: pr_tests: @@ -40,7 +41,7 @@ jobs: - name: Create report directory # Creates the temporary directory that is used to store the test report run: | - mkdir -p $REPORT_DIRECTORY + mkdir -p env.REPORT_DIRECTORY - name: Check formatting with black run: black . --check @@ -59,7 +60,7 @@ jobs: - name: Test with pytest run: | - pytest tests/pre-merge -c tests/offline.ini --cov=geti_sdk --html=$REPORT_DIRECTORY/$PREMERGE_TEST_REPORT --self-contained-html --cov-report html:$REPORT_DIRECTORY/coverage + pytest tests/pre-merge -c tests/offline.ini --cov=geti_sdk --html=env.REPORT_DIRECTORY/env.PREMERGE_TEST_REPORT --self-contained-html --cov-report html:env.REPORT_DIRECTORY/coverage - name: Upload test report for pre-merge tests # Publish the test report to github @@ -73,4 +74,4 @@ jobs: # Remove temporary report directory if: ${{ always() }} run: | - rm -r $REPORT_DIRECTORY + rm -r env.REPORT_DIRECTORY From 4854de120919aa23599272f7137e5ebe4d1e8d8a Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 14:16:13 +0100 Subject: [PATCH 6/8] solve windows encoding problem Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index 99622700..db43450a 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -19,9 +19,9 @@ jobs: runs-on: ${{ matrix.target[0] }} strategy: matrix: - target: [[ubuntu-latest, '3.8'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] # [ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], + target: [[ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] fail-fast: true - max-parallel: 5 + max-parallel: 1 steps: - name: Checkout repository uses: actions/checkout@v4 From fbaf2df8dc085b92d3e84cedbcf2a0f97a3d8255 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 17:05:25 +0100 Subject: [PATCH 7/8] enable parallelism Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index db43450a..09972be6 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -16,12 +16,19 @@ env: jobs: pr_tests: - runs-on: ${{ matrix.target[0] }} + runs-on: ${{ matrix.os }} strategy: matrix: - target: [[ubuntu-latest, '3.8'], [ubuntu-latest, '3.9'], [ubuntu-latest, '3.10'], [windows-latest, '3.8'], [windows-latest, '3.9'], [macos-latest, '3.8'], [macos-latest, '3.9']] + python-version: ['3.8', '3.9', '3.10'] + os: [ubuntu-latest, windows-latest, macos-latest] + exclude: + - os: macos-latest + python-version: 10 + - os: windows-latest + python-version: 10 fail-fast: true - max-parallel: 1 + max-parallel: 3 + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -31,7 +38,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.target[1] }} + python-version: ${{ matrix.python-version }} - name: Install package with dev and notebook requirements run: | From 2a75e22c70ffbe14432a158ea04d4a1ea72a6e0c Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Tue, 13 Feb 2024 17:08:45 +0100 Subject: [PATCH 8/8] fix typo Signed-off-by: Igor Davidyuk --- .github/workflows/pre-merge-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml index 09972be6..14222c69 100644 --- a/.github/workflows/pre-merge-tests.yml +++ b/.github/workflows/pre-merge-tests.yml @@ -23,9 +23,9 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] exclude: - os: macos-latest - python-version: 10 + python-version: '3.10' - os: windows-latest - python-version: 10 + python-version: '3.10' fail-fast: true max-parallel: 3