CI: fixed build and test, polished them, prepare for publishing

.github/workflows/package.yml

@@ -0,0 +1,288 @@
+name: Package
+
+# This workflow handles:
+# 1. Code quality checks for both Rust and Python
+# 2. Building the main Python package
+# 3. Building Rust service extensions (memory, s3)
+# 4. Verifying package installation and imports
+# Future: PyPI publishing on tags
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
+
+jobs:
+  check-rust:
+    name: Rust Checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Cache Rust dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Check format
+        run: cargo fmt --all -- --check
+
+      - name: Check clippy
+        run: cargo clippy --all-targets --all-features -- -D warnings
+
+  check-python:
+    name: Python Checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Check format
+        run: ruff format . --check
+
+      - name: Check style
+        run: ruff check .
+
+  build-main:
+    name: Build Main Package
+    needs: [check-rust, check-python]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Create dist directory
+        run: mkdir -p dist
+
+      - name: Build main package
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build --outdir dist/
+
+      - name: List dist contents
+        run: ls -la dist/
+
+      - name: Upload main artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist-main
+          path: dist/*
+          if-no-files-found: error
+
+  build-services:
+    name: Build Service Packages
+    needs: [build-main]
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04]
+        target: [x86_64]
+        service: [memory, s3]
+        exclude:
+          - os: ubuntu-24.04
+            target: aarch64
+        include:
+          - os: ubuntu-24.04
+            target: x86_64
+            platform: manylinux2014
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install build dependencies
+        run: pip install maturin
+
+      - name: Build service package
+        id: build-service
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.platform }}
+          container: quay.io/pypa/manylinux2014_x86_64
+          args: --release --out dist --interpreter python3.11
+          command: build
+          working-directory: crates/service-${{ matrix.service }}
+
+      - name: List all wheels
+        run: |
+          echo "=== All wheel locations ==="
+          find . -name "*.whl" -exec ls -l {} \;
+          echo "=== Service directory contents ==="
+          ls -la crates/service-${{ matrix.service }}/dist/
+
+      - name: Upload service artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.service }}
+          path: crates/service-${{ matrix.service }}/dist/*.whl
+          if-no-files-found: error
+
+  verify-install:
+    name: Verify Installation
+    needs: [build-services]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: dist
+
+      - name: List downloaded artifacts
+        run: |
+          echo "=== All artifacts ==="
+          find dist -type f
+          echo "=== Service artifacts ==="
+          find dist/dist-ubuntu-24.04-x86_64-memory dist/dist-ubuntu-24.04-x86_64-s3 -type f
+
+      - name: Prepare dist directory
+        run: |
+          mkdir -p dist_combined
+          # Copy service wheels first (only cp311 wheels)
+          cp dist/dist-ubuntu-24.04-x86_64-memory/*cp311*.whl dist_combined/
+          cp dist/dist-ubuntu-24.04-x86_64-s3/*cp311*.whl dist_combined/
+          # Copy main package
+          cp dist/dist-main/*.whl dist_combined/
+          cp dist/dist-main/*.tar.gz dist_combined/
+          echo "=== Contents of dist_combined ==="
+          ls -la dist_combined/
+
+      - name: Install packages in correct order
+        run: |
+          python -m pip install --upgrade pip
+          echo "=== Installing service packages ==="
+          # Install memory service (only cp311 wheels)
+          for whl in dist_combined/opendalfs_service_memory*cp311*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing memory service: $whl"
+              pip install "$whl"
+            fi
+          done
+          # Install s3 service (only cp311 wheels)
+          for whl in dist_combined/opendalfs_service_s3*cp311*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing s3 service: $whl"
+              pip install "$whl"
+            fi
+          done
+          echo "=== Installing main package ==="
+          pip install dist_combined/opendalfs-*.whl
+
+      - name: Verify imports
+        run: |
+          echo "=== Installed packages ==="
+          pip list | grep opendalfs
+          echo "=== Verifying imports ==="
+          python -c "import opendalfs"
+          python -c "import opendalfs_service_memory"
+          python -c "import opendalfs_service_s3"
+
+      - name: Test full installation
+        run: |
+          # Clean previous installation
+          pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3
+          # Install with all extras (use exact filename)
+          echo "=== Installing with all extras ==="
+          # First install service packages
+          for whl in dist_combined/opendalfs_service_*manylinux_2_17*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing service: $whl"
+              pip install "$whl"
+            fi
+          done
+          # Then install main package with all extras
+          MAIN_WHEEL=$(find dist_combined -name "opendalfs-*.whl")
+          echo "Installing main package with extras: $MAIN_WHEEL"
+          pip install "$MAIN_WHEEL[all]"
+          # Verify imports
+          echo "=== Verifying imports ==="
+          python -c "import opendalfs"
+          python -c "import opendalfs_service_memory"
+          python -c "import opendalfs_service_s3"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3 || true
+          rm -rf dist dist_combined
+
+  # PyPI publishing job (commented out for future use)
+  # publish:
+  #   name: Publish to PyPI
+  #   needs: verify-install
+  #   runs-on: ubuntu-24.04
+  #   # Only run on tags
+  #   if: startsWith(github.ref, 'refs/tags/')
+  #   
+  #   steps:
+  #     - name: Download all artifacts
+  #       uses: actions/download-artifact@v3
+  #       with:
+  #         path: dist
+  #
+  #     - name: Prepare dist directory
+  #       run: |
+  #         mkdir -p dist_combined
+  #         cp -r dist/*/* dist_combined/
+  #
+  #     - name: Publish to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+  #       with:
+  #         password: ${{ secrets.PYPI_API_TOKEN }}
+  #         packages-dir: dist_combined/
+  #
+  # Required secrets for PyPI publishing:
+  # - PYPI_API_TOKEN: API token from PyPI
+  #   Get it from: https://pypi.org/manage/account/token/
+  #   Add it to: https://github.com/fsspec/opendalfs/settings/secrets/actions