Migrate to uv for package management (#683)

* Update pyproject.toml for use with uv instead of poetry * Update CONTRIBUTING.md * More pyproject.toml tweaks * Attempt to change circle CI to work with this * Tweak installation code blocks * Update docs/setup_insiders to use uv * Update dataset examples for uv instead of poetry * Fix example test for classification * Fix example test for named_entity_recognition * Test out docs building without ssh key * Fix typing in named_entity_recognition example * Remove insiders docs sources * Continue using poetry for workflows * Add extra index url for installing torch * Include explicit version tag for mkdocstring-python * Remove python version mapping from poetry * Remove tool.poetry.scripts * Remove poetry from github workflows * update docs pt 1 * Get stuff maybe working on both 3.8 and 3.12? * Remove poetry from dataset docs pt2 * Remove transitive torch dependency for text_summarization * Update dataset example READMEs with uv instructions * Update workflows docs to use uv as well (outside of example projects) * Add docs warning * Re-add project.scripts * Make sure to update the version when packaging for release * Add torch cpu blurb to relevant example READMEs
kolenaIO · Sep 18, 2024 · 054090c · 054090c
1 parent cc374bc
commit 054090c
Show file tree

Hide file tree

Showing 52 changed files with 641 additions and 590 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -1,8 +1,5 @@
 version: 2.1
 
-orbs:
-  codecov: codecov/codecov@3.2.4
-
 parameters:
   workflow:
     type: boolean
@@ -35,8 +32,6 @@ executors:
     docker:
       - image: cimg/python:<< parameters.python-version >>
     resource_class: << parameters.resource-class >>
-    environment:
-      POETRY_CACHE_DIR: /home/circleci/project/.poetry
 
 jobs:
   ci-base:
@@ -52,31 +47,12 @@ jobs:
       python-version: << parameters.python-version >>
     steps:
       - checkout
-      - restore_cache:
-          key: &poetry-cache poetry-cache-<< parameters.python-version >>-<< parameters.extras >>-{{ checksum "pyproject.toml" }}
-      - when:
-          condition:
-            equal: [ "none", << parameters.extras >> ]
-          steps:
-            - run:
-                name: Install Dependencies
-                command: |
-                  poetry config installer.max-workers 10
-                  poetry install -vv --no-ansi
-      - when:
-          condition:
-            equal: [ "metrics", << parameters.extras >> ]
-          steps:
-            - run:
-                name: Install Dependencies
-                command: |
-                  poetry config installer.max-workers 10
-                  poetry install -vv --all-extras --no-ansi
-      - save_cache:
-          key: *poetry-cache
-          paths:
-            - /home/circleci/project/.poetry/virtualenvs
-            - poetry.lock
+      - run: &install_uv
+          name: Install uv
+          command: pip install uv
+      - run: &setup_env
+          name: Setup environment
+          command: uv venv --python << parameters.python-version >> && source .venv/bin/activate
 
   docs:
     parameters:
@@ -91,9 +67,8 @@ jobs:
       python-version: << parameters.python-version >>
     steps:
       - checkout
-      - restore_cache:
-          name: Restore Poetry cache
-          key: poetry-cache-<< parameters.python-version >>-<< parameters.extras >>-{{ checksum "pyproject.toml" }}
+      - run: *install_uv
+      - run: *setup_env
       - restore_cache:
           name: Restore MkDocs cache
           key: mkdocs-cache
@@ -107,14 +82,14 @@ jobs:
           command: git lfs pull
       - run:
           name: Build documentation without 'insiders' packages
-          command: poetry run mkdocs build
+          command: uv run mkdocs build
       - add_ssh_keys: # add github.com SSH key fingerprint to access private repo forks
           fingerprints: [ "15:8b:d4:ac:1f:cd:2f:d3:92:a7:4c:aa:46:81:0b:7d" ]
       - run:
           name: Build documentation with 'insiders' packages
           command: |
             ./docs/setup_insiders.sh
-            poetry run mkdocs build --config-file mkdocs.insiders.yml
+            uv run mkdocs build --config-file mkdocs.insiders.yml
       - save_cache:
           key: mkdocs-cache
           paths:
@@ -133,9 +108,27 @@ jobs:
       python-version: << parameters.python-version >>
     steps:
       - checkout
-      - restore_cache:
-          key: poetry-cache-<< parameters.python-version >>-<< parameters.extras >>-{{ checksum "pyproject.toml" }}
-      - run: poetry run python3 -c 'import kolena'
+      - run: *install_uv
+      - run: *setup_env
+      - when: &install_without_extras
+          condition:
+            equal: [ "none", << parameters.extras >> ]
+          steps:
+            - run:
+                name: Install Dependencies
+                command: uv pip install -r pyproject.toml --verbose
+                env:
+                  UV_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu/"
+      - when: &install_with_extras
+          condition:
+            equal: [ "metrics", << parameters.extras >> ]
+          steps:
+            - run:
+                name: Install Dependencies
+                command: uv pip install --all-extras -r pyproject.toml --verbose
+                env:
+                  UV_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu/"
+      - run: uv run python3 -c 'import kolena'
       # TODO: fix underlying mypy issues with Python>3.9 rather than skipping
       - when:
           condition:
@@ -144,34 +137,21 @@ jobs:
                 pattern: "^3\\.(10|11|12).*$"
                 value: << parameters.python-version >>
           steps:
-            - run: poetry run pre-commit run -a
+            - run: uvx pre-commit run -a
       - when:
           condition:
             equal: [ "none", << parameters.extras >> ]
           steps:
             - run:
                 name: Run unit tests
-                command: poetry run pytest --cache-clear -m 'not metrics' -vv --cov=kolena --cov-branch tests/unit
+                command: uv run pytest --cache-clear -m 'not metrics' -vv --cov=kolena --cov-branch tests/unit
       - when:
           condition:
             equal: [ "metrics", << parameters.extras >> ]
           steps:
             - run:
                 name: Run metrics unit tests
-                command: poetry run pytest --cache-clear -m 'metrics' -vv --cov=kolena --cov-branch tests/unit
-      - when:
-          # Generate coverage only from one Python version
-          condition:
-            matches:
-              pattern: "^3\\.9.*$"
-              value: << parameters.python-version >>
-          steps:
-            - run:
-                name: Coverage
-                command: poetry run coverage xml --data-file .coverage
-            - codecov/upload:
-                file: coverage.xml
-                flags: integration
+                command: uv run pytest --cache-clear -m 'metrics' -vv --cov=kolena --cov-branch tests/unit
 
   integration-test:
     parameters:
@@ -203,8 +183,10 @@ jobs:
                 name: skip job
                 command: circleci-agent step halt
       - checkout
-      - restore_cache:
-          key: poetry-cache-<< parameters.python-version >>-<< parameters.extras >>-{{ checksum "pyproject.toml" }}
+      - run: *install_uv
+      - run: *setup_env
+      - when: *install_without_extras
+      - when: *install_with_extras
       - run:
           name: Set KOLENA_TOKEN with round robin
           command: |
@@ -229,29 +211,15 @@ jobs:
                     TESTFILES=$(find tests/integration/$TEST_GROUP -name "test_*.py" |
                       circleci tests split --split-by=timings --timings-type=filename)
                   fi
-                  poetry run pytest --cache-clear -vv --durations=10 --cov=kolena --cov-branch --ignore=examples \
+                  uv run pytest --cache-clear -vv --durations=10 --cov=kolena --cov-branch --ignore=examples \
                     -o junit_family=legacy --junitxml=test-results/result.xml $TESTFILES
       - when:
           condition:
             equal: [ "metrics", << parameters.extras >> ]
           steps:
             - run:
                 name: Run experimental metrics integration tests
-                command: poetry run pytest --cache-clear -m 'metrics' -vv --cov=kolena --cov-branch tests/integration/
-      - when:
-          # Generate coverage only from one python version
-          condition:
-            matches:
-              pattern: "^3\\.9.*$"
-              value: << parameters.python-version >>
-          steps:
-            - run:
-                name: Coverage
-                command: poetry run coverage xml --data-file .coverage
-            - codecov/upload:
-                file: coverage.xml
-            - store_test_results:
-                path: test-results
+                command: uv run pytest --cache-clear -m 'metrics' -vv --cov=kolena --cov-branch tests/integration/
 
   integration-test-dataset:
     parameters:
@@ -287,8 +255,10 @@ jobs:
                 name: skip job
                 command: circleci-agent step halt
       - checkout
-      - restore_cache:
-          key: poetry-cache-<< parameters.python-version >>-<< parameters.extras >>-{{ checksum "pyproject.toml" }}
+      - run: *install_uv
+      - run: *setup_env
+      - when: *install_without_extras
+      - when: *install_with_extras
       - run:
           name: Set KOLENA_TOKEN with round robin
           command: |
@@ -300,22 +270,8 @@ jobs:
           command: |
             TESTFILES=$(circleci tests glob tests/integration/dataset/**/test_*.py |
                 circleci tests split --split-by=timings --timings-type=filename)
-            poetry run pytest --cache-clear -vv --durations=10 --cov=kolena --cov-branch --ignore=examples \
+            uv run pytest --cache-clear -vv --durations=10 --cov=kolena --cov-branch --ignore=examples \
               -o junit_family=legacy --junitxml=test-results/result.xml $TESTFILES
-      - when:
-          # Generate coverage only from one python version
-          condition:
-            matches:
-              pattern: "^3\\.9.*$"
-              value: << parameters.python-version >>
-          steps:
-            - run:
-                name: Coverage
-                command: poetry run coverage xml --data-file .coverage
-            - codecov/upload:
-                file: coverage.xml
-            - store_test_results:
-                path: test-results
 
   example-test-dataset:
     parameters:
@@ -334,19 +290,19 @@ jobs:
     steps:
       - checkout:
           path: ~/project
-      - run: |
-          poetry config installer.max-workers 10
-          poetry install -vv --no-ansi
+      - run: *install_uv
+      - run: *setup_env
+      - run: uv pip install -r pyproject.toml --verbose
       - run:
           name: Run pre-commit checks
-          command: poetry run pre-commit run -a
+          command: uvx pre-commit run -a
       - run:
           name: Run << parameters.subproject >> (Python << parameters.python-version >>) integration tests
           command: |
             token=KOLENA_TOKEN_$((CIRCLE_BUILD_NUM % << pipeline.parameters.token_count >>))
             echo "Using $token"
             export KOLENA_TOKEN=${!token}
-            poetry run pytest -vv tests
+            uv run pytest -vv tests
 
   example-test-workflow:
     parameters:

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -38,25 +38,24 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
 
-      - name: Install Poetry
-        run: pip install poetry==1.5.1
+      - uses: astral-sh/setup-uv@v2
+
+      - name: Setup environment
+        run: uv venv && source .venv/bin/activate
 
       - name: Fetch tags to enable autoversioning
         run: git fetch --prune --unshallow --tags
 
-      - name: Update package version to PEP 440-compliant production release tag
-        run: poetry version $(git describe --tags --abbrev=0)
-
-      - name: Install Poetry dependencies
-        run: poetry install
+      - name: Install dependencies
+        run: uv pip install --all-extras -r pyproject.toml
 
       - name: Install docs 'insiders' dependencies
         run: ./docs/setup_insiders.sh
 
       - name: Publish 'kolena' documentation to trunk (S3)
         if: inputs.environment != 'production'
         run: |
-          poetry run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
+          uv run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
           aws s3 sync ./site "s3://trunk-docs.kolena.io" --delete
           aws cloudfront create-invalidation --no-cli-pager --distribution-id ${{ secrets.TRUNK_DOC_DISTRIBUTION_ID }} --paths "/*"
           aws s3 sync ./site "s3://trunk-docs.kolena.com" --delete
@@ -65,7 +64,7 @@ jobs:
       - name: Publish 'kolena' documentation to production (S3)
         if: inputs.environment == 'production'
         run: |
-          poetry run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
+          uv run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
           aws s3 sync ./site "s3://docs.kolena.io" --delete
           aws cloudfront create-invalidation --no-cli-pager --distribution-id ${{ secrets.DOC_DISTRIBUTION_ID }} --paths "/*"
           aws s3 sync ./site "s3://docs.kolena.com" --delete

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -31,28 +31,27 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
 
-      - name: Install Poetry
-        run: pip install poetry==1.5.1
+      - uses: astral-sh/setup-uv@v2
 
       - name: Fetch tags to enable autoversioning
         run: git fetch --prune --unshallow --tags
 
-      - name: Update package version to PEP 440-compliant production release tag
-        run: poetry version $(git describe --tags --abbrev=0)
-
       - name: Install dependencies
-        run: poetry install
+        run: uv pip install --all-extras -r pyproject.toml
 
       - name: Run pre-commit checks
-        run: poetry run pre-commit run -a
+        run: uvx pre-commit run -a
+
+      - name: Update package version to PEP 440-compliant release tag
+        run: sed -i "s/0.999.0/$(git describe --tags --abbrev=0)/g" pyproject.toml
 
       - name: Build 'kolena' Python package
-        run: poetry build --format=sdist
+        run: uv build --format=sdist
 
       - name: Publish 'kolena' documentation to production (S3)
         run: |
           ./docs/setup_insiders.sh
-          poetry run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
+          uv run mkdocs build --verbose --strict --config-file mkdocs.insiders.yml
           aws s3 sync ./site "s3://docs.kolena.io" --delete
           aws cloudfront create-invalidation --no-cli-pager --distribution-id ${{ secrets.DOC_DISTRIBUTION_ID }} --paths "/*"
           aws s3 sync ./site "s3://docs.kolena.com" --delete
@@ -91,8 +90,8 @@ jobs:
           # update first instance of 'kolena' to 'kolena-client' in pyproject.toml, kolena/__init__.py (package name)
           sed -i '0,/kolena/{s/kolena/kolena-client/}' pyproject.toml kolena/__init__.py
 
-          poetry install
-          poetry build --format=sdist
+          uv pip install --all-extras -r pyproject.toml
+          uv build --format=sdist
 
       - name: "[backcompat] Install twine for package distribution on CodeArtifact"
         run: pip install twine

diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@
 *.code-workspace
 
 .DS_Store
+.venv/
 
 __pycache__/
 .ipynb_checkpoints/
@@ -22,3 +23,4 @@ dist/
 site
 
 poetry.lock
+uv.lock