Use maturin as a build backend and improve github workflows

.github/actions/bump_version/action.yml

@@ -0,0 +1,27 @@
+# This action is called from PyPi and Rust release workflows to update Cargo.toml with release version.
+# It will only be successfull, if release tag, which triggered the workflow will match
+# package version in Cargo.toml (sanity check).
+name: 'Update package version to release version'
+description: 'This action bumps package version in Cargo.toml file, matching the release tag.'
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/checkout@v4
+      # To fetch all the tags and history
+      with:
+        fetch-depth: 0
+
+    - name: Bump version in Cargo.toml
+      run: |
+        python .github/actions/bump_version/bump_version.py --target Cargo.toml "${{ github.ref_name }}"
+      shell: bash
+
+    - name: Check version in Cargo.toml matches Release tag
+      run: |
+        VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')
+        if [ "${GITHUB_REF#refs/tags/}" != "$VERSION" ]; then
+          echo "Version mismatch: Cargo.toml ($VERSION) doesn't match Release tag (${GITHUB_REF#refs/tags/})"
+          exit 1
+        fi
+      shell: bash

.github/workflows/build_wheels.yml

@@ -0,0 +1,232 @@
+name: Build Wheels
+
+on:
+  workflow_call:
+    inputs:
+      auto_bump:
+        description: 'Bump version automatically'
+        type: boolean
+        required: false
+        default: false
+
+jobs:
+  linux:
+    name: Build ${{ matrix.platform.runner}} ${{ matrix.platform.target }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          # older ubuntu to avoid messing with glibc version
+          - runner: ubuntu-22.04
+            target: x86_64
+            manylinux: auto
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+          - runner: ubuntu-22.04
+            target: aarch64
+            manylinux: manylinux_2_28
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - run: pip install -U twine
+
+      - name: Install required packages
+        run: |
+          sudo apt update
+          sudo apt install pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu -qy
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: ${{ matrix.platform.target}}-unknown-linux-gnu
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Bump version in Cargo.toml
+        if: ${{ inputs.auto_bump }}
+        uses: ./.github/actions/bump_version
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --interpreter ${{ matrix.platform.interpreter }}
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: ${{ matrix.platform.manylinux }}
+
+      - name: Validate Python package distributions
+        run: twine check --strict dist/*
+
+      - name: Install built wheel
+        if: matrix.platform.target == 'x86_64'
+        run: |
+          pip install outlines_core --no-index --find-links dist --force-reinstall
+          python -c "import outlines_core"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/*.whl
+          name: wheels-linux-${{ matrix.platform.target }}
+
+  windows:
+    name: Build ${{ matrix.platform.runner}} ${{ matrix.platform.target }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: windows-latest
+            target: x86
+            alias-target: i686-pc-windows-msvc
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+          - runner: windows-latest
+            target: x64
+            alias-target: x86_64-pc-windows-msvc
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          architecture: ${{ matrix.platform.target }}
+
+      - run: pip install -U twine
+
+      - name: Install required packages
+        # rustls requires aws-lc-sys, which FFI bindings to AWS-LC (AWS Libcrypto)
+        # aws-lc-sys requires nasm for compilation on windows
+        run: |
+          choco install nasm
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: ${{ matrix.platform.alias-target}}
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Bump version in Cargo.toml
+        if: ${{ inputs.auto_bump }}
+        uses: ./.github/actions/bump_version
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --interpreter ${{ matrix.platform.interpreter }}
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: auto
+
+      - name: Validate Python package distributions
+        run: twine check --strict dist/*
+
+      - name: Install built wheel
+        run: |
+          pip install outlines_core --no-index --find-links dist --force-reinstall
+          python -c "import outlines_core"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/*.whl
+          name: wheels-windows-${{ matrix.platform.target }}
+
+  macos:
+    name: Build ${{ matrix.platform.runner}} ${{ matrix.platform.target }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: macos-14
+            target: x86_64
+            macos_version: "14.0"
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+          - runner: macos-14
+            target: aarch64
+            macos_version: "14.0"
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+          - runner: macos-15
+            target: x86_64
+            macos_version: "15.0"
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+          - runner: macos-15
+            target: aarch64
+            macos_version: "15.0"
+            interpreter: "3.9 3.10 3.11 3.12 3.13"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - run: pip install -U twine
+
+      - name: Set macOS version
+        run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.platform.macos_version }}" >> $GITHUB_ENV
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: ${{ matrix.platform.target}}-apple-darwin
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Bump version in Cargo.toml
+        if: ${{ inputs.auto_bump }}
+        uses: ./.github/actions/bump_version
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --interpreter ${{ matrix.platform.interpreter }}
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: auto
+
+      - name: Validate Python package distributions
+        run: twine check --strict dist/*
+
+      - name: Install built wheel
+        if: matrix.platform.target == 'aarch64'
+        run: |
+          pip install outlines_core --no-index --find-links dist --force-reinstall
+          python -c "import outlines_core"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/*.whl
+          name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
+
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Bump Cargo version
+        if: ${{ inputs.auto_bump }}
+        uses: ./.github/actions/bump_version
+
+      - name: Build sdist with Maturin
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/*.tar.gz
+          name: sdist

.github/workflows/dry_run_publish.yml

@@ -0,0 +1,43 @@
+name: Dry-run Publish
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  # TODO: remove
+  pull_request:
+    branches: [main]
+
+jobs:
+  build_wheels:
+    # TODO: switch to main
+    uses: dottxt-ai/outlines-core/.github/workflows/build_wheels.yml@maturin
+
+  dry_run:
+    needs: build_wheels
+    name: Dry-run for publishing to PyPI and crates.io
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          merge-multiple: true
+          path: dist
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: List downloaded artifacts
+        run: ls -lh dist/
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Dry-run publish to crates.io
+        run: cargo publish --dry-run

.github/workflows/publish.yml

@@ -0,0 +1,61 @@
+name: Publish PyPi & crates.io
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag"
+        required: true
+  release:
+    types:
+      - created
+
+jobs:
+  build_wheels:
+    # TODO: switch to main
+    uses: dottxt-ai/outlines-core/.github/workflows/build_wheels.yml@maturin
+    with:
+      auto_bump: true
+
+  release:
+    needs: build_wheels
+    name: Publish PyPI and crates.io
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          merge-multiple: true
+          path: dist
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Bump version in Cargo.toml
+        uses: ./.github/actions/bump_version
+
+      - name: Dry-run publish to crates.io
+        run: cargo publish --dry-run
+
+      - name: Publish to PyPI
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_SECRET}}
+        with:
+          command: upload
+          args: --non-interactive --skip-existing dist/*
+
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        # We need --allow-dirty since we dynamically change the version in Cargo.toml
+        run: cargo publish --allow-dirty