Merge pull request #153 from thomasmarwitz/add-pixi

Port to pixi and QuantCo Oss copier template

.copier-answers.yml

@@ -0,0 +1,12 @@
+# This file is managed by Copier; DO NOT EDIT OR REMOVE.
+_commit: v0.2.1
+_src_path: https://github.com/quantco/copier-template-python-open-source
+add_autobump_workflow: true
+author_email: minimalkv@uwekorn.com
+author_name: Data Engineering Collective
+github_url: https://github.com/data-engineering-collective/minimalkv
+github_user: xhochy
+minimal_python_version: py39
+project_short_description: A key-value storage for binary data, support many backends.
+project_slug: minimalkv
+use_devcontainer: false

.gitattributes

@@ -0,0 +1 @@
+pixi.lock linguist-language=YAML linguist-generated=true

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,5 +2,7 @@
 Thank you for pull request.
 Below are a few things we ask you kindly to self-check before getting a review. Remove checks that are not relevant.
 -->
+
 Checklist
-* [ ] Added a `docs/changes.rst` entry
+
+- [ ] Added a `docs/changes.rst` entry

.github/workflows/build.yml

@@ -0,0 +1,54 @@
+name: Build
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          fetch-depth: 0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: build
+      - name: Build project
+        run: pixi run -e build build-wheel
+      - name: Check package
+        run: pixi run -e build check-wheel
+      - name: Upload package
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: artifact
+          path: dist/*
+
+  release:
+    name: Publish package
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: [build]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment: pypi
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: artifact
+          path: dist
+      - name: Publish package on TestPyPi
+        uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          user: __token__
+          password: ${{ secrets.pypi_password }}
+      - name: Publish package on PyPi
+        uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_password }}

.github/workflows/ci.yml

@@ -1,35 +1,54 @@
 name: CI
 on: [push, pull_request, workflow_dispatch]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
+  pre-commit:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: default lint
+      - name: pre-commit
+        run: pixi run pre-commit-run --color=always --show-diff-on-failure
+
   unittest:
-    name: Unit tests - ${{ matrix.PYTHON_VERSION }}
+    name: Unit tests - ${{ matrix.environment }}
     runs-on: ubuntu-latest
     permissions:
       contents: "read"
       id-token: "write"
     strategy:
       fail-fast: false
       matrix:
-        PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        environment:
+          - py39
+          - py310
+          - py311
+          - py312
     services:
       postgres:
         image: postgres:11
         env:
           POSTGRES_DB: minimalkv_test
           POSTGRES_PASSWORD: minimalkv_test
           POSTGRES_USER: minimalkv_test
-    env:
-      SIMPLEKV_CI: 1
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up Conda env
-        uses: mamba-org/setup-micromamba@068f1ab4b37ed9b3d9f73da7db90a0cda0a48d29
+      - name: Checkout branch
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
         with:
-          environment-file: environment.yml
-          cache-environment: true
-          create-args: >-
-            python=${{ matrix.PYTHON_VERSION }}
+          environments: ${{ matrix.environment }}
+      - name: Install repository
+        run: pixi run -e ${{ matrix.environment }} postinstall
       # For some reason we can't check for $ACTIONS_ID_TOKEN_REQUEST_URL
       # from the `if` condition in the next step.
       # See this commit: https://github.com/simonbohnen/minimalkv/commit/775c74c1b1248245d66c2be69476e18acda9f2fe
@@ -57,62 +76,42 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::211125346859:role/github-poweruser
           aws-region: eu-central-1
-        # We set an env variable according to the result of the check
-        # to only allow skipping of aws integration test when in fork.
-        # When being run in the base repo, the aws integration test should always be executed.
+
+      # This is a work around as long as S3FSStore has the side effect of setting env variables
+      # to provide authentication. This can safely be removed as soon as this side effect is gone
       - name: Remap AWS Environment Variables
         if: steps.check-id-token.outcome == 'success'
         run: |
           echo "ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV
           echo "SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
           echo "SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }}" >> $GITHUB_ENV
-        # This is a work around as long as S3FSStore has the side effect of setting env variables
-        # to provide authentication. This can savely be removed as soon as this side effect is gone
+      # We set an env variable according to the result of the check
+      # to only allow skipping of aws integration test when in fork.
+      # When being run in the base repo, the aws integration test should always be executed.
       - name: Check whether the workflow runs in a fork
         run: echo "CI_IN_FORK=${{ github.event.pull_request && github.repository != github.event.pull_request.head.repo.full_name }}" >> $GITHUB_ENV
       - name: "Run setup: install package & start docker container"
         shell: bash -x -l {0}
         run: |
-          pip install --no-deps .
-          docker-compose up -d
+          pixi run docker compose up -d
           while ! docker exec mysql mysqladmin status -h 127.0.0.1 -u minimalkv_test --password=minimalkv_test; \
             do sleep 3; done
-      - name: Run the unittests
-        shell: bash -l {0}
-        run: |
-          pytest -n auto --dist loadfile -rs --cov=minimalkv --cov-report=xml
-      - name: Build the package
-        shell: bash -x -l {0}
-        run: |
-          python -m build
-      - name: Typecheck the codebase
-        shell: bash -x -l {0}
-        run: mypy minimalkv
-      - name: Publish package
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && matrix.PYTHON_VERSION == '3.10'
-        uses: pypa/gh-action-pypi-publish@v1.12.3
-        with:
-          user: __token__
-          password: ${{ secrets.pypi_password }}
+      - name: Run pytest
+        run: pixi run -e ${{ matrix.environment }} pytest -n auto --dist loadfile -rs --cov=minimalkv --cov-report=xml --color=yes
       - uses: codecov/codecov-action@v5
         with:
           file: ./coverage.xml
 
-  pre-commit-checks:
-    name: "Pre-commit checks"
+  mypy:
     runs-on: ubuntu-latest
-    env:
-      PRE_COMMIT_USE_MICROMAMBA: 1
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v4
-      - name: Set up micromamba
-        uses: mamba-org/setup-micromamba@068f1ab4b37ed9b3d9f73da7db90a0cda0a48d29
-      - name: Add micromamba to GITHUB_PATH
-        run: echo "${HOME}/micromamba-bin" >> "$GITHUB_PATH"
-      - name: Install Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
         with:
-          python-version: "3.10"
-      - name: Run pre-commit checks
-        uses: pre-commit/action@v3.0.1
+          environments: py39 # as in pyproject.toml
+      - name: Install repository
+        run: pixi run -e py39 postinstall
+      - name: Run mypy
+        run: pixi run -e py39 mypy minimalkv

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ main ]
+    branches: [main]
   schedule:
-    - cron: '31 21 * * 1'
+    - cron: "31 21 * * 1"
 
 jobs:
   analyze:
@@ -28,21 +28,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: ["python"]
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/codeql.yml

@@ -14,12 +14,12 @@ name: "CodeQL"
 on:
   workflow_dispatch:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "main" ]
+    branches: ["main"]
   schedule:
-    - cron: '25 17 * * 5'
+    - cron: "25 17 * * 5"
 
 jobs:
   analyze:
@@ -33,43 +33,42 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: ["python"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
 
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
 
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh
 
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
-      with:
-        category: "/language:${{matrix.language}}"
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/pre-commit-autoupdate.yml

@@ -18,22 +18,16 @@ jobs:
         # We need to checkout with SSH here to have actions run on the PR.
         with:
           ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - name: Set up Conda env
-        uses: mamba-org/setup-micromamba@068f1ab4b37ed9b3d9f73da7db90a0cda0a48d29
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
         with:
-          environment-name: pre-commit
-          create-args: >-
-            -c
-            conda-forge
-            pre-commit
-            mamba
+          environments: default lint
       - name: Update pre-commit hooks and run
         id: versions
-        env:
-          PRE_COMMIT_USE_MAMBA: 1
         run: |
-          pre-commit autoupdate
-          pre-commit run -a || true
+          pixi run pre-commit-update
+          pixi run pre-commit-run --color=always --show-diff-on-failure || true
+
       - uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f
         with:
           commit-message: "Auto-update pre-commit hooks"