From a4222adfda6c385f4b40c41009acb9e5d292e119 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:06:49 +0800
Subject: [PATCH 01/18] elide the unnecessary lifetimes

to make clippy happy
---
 crates/core/src/fs.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/crates/core/src/fs.rs b/crates/core/src/fs.rs
index a80be71..a34c2a0 100644
--- a/crates/core/src/fs.rs
+++ b/crates/core/src/fs.rs
@@ -96,7 +96,7 @@ impl OpendalFileSystem {
         })
     }
 
-    fn mkdirs<'p>(&self, py: Python<'p>, path: &str, exist_ok: bool) -> PyResult<()> {
+    fn mkdirs(&self, py: Python, path: &str, exist_ok: bool) -> PyResult<()> {
         let path = normalize_path(path);
         let exists = self.check_path_exists(py, &path)?;
         if exists && !exist_ok {
@@ -166,7 +166,7 @@ impl OpendalFileSystem {
         })
     }
 
-    fn check_path_exists<'p>(&self, py: Python<'p>, path: &str) -> PyResult<bool> {
+    fn check_path_exists(&self, py: Python, path: &str) -> PyResult<bool> {
         let path = normalize_path(path);
         let op = self.op.clone();
 

From 02d15f84026245a773bfb5f5d7bb73085980a2ae Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:12:28 +0800
Subject: [PATCH 02/18] ci: fix test workflow

---
 .github/workflows/test.yml | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7379a38..34d383d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,24 +10,6 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
-    services:
-      minio:
-        image: minio/minio
-        ports:
-          - 9000:9000
-          - 9001:9001
-        env:
-          MINIO_ROOT_USER: minioadmin
-          MINIO_ROOT_PASSWORD: minioadmin
-        options: >-
-          --health-cmd "curl -f http://localhost:9000/minio/health/live"
-          --health-interval 30s
-          --health-timeout 20s
-          --health-retries 3
-        volumes:
-          - /tmp/data:/data
-        command: server /data --console-address ":9001"
-
     steps:
     - uses: actions/checkout@v3
 
@@ -36,6 +18,12 @@ jobs:
       with:
         python-version: '3.11'
 
+    - name: Start MinIO
+      run: |
+        docker compose -f tests/docker/docker-compose.yml up -d
+        # Wait for MinIO to be ready
+        sleep 10
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -51,4 +39,8 @@ jobs:
         pytest -v --cov=opendalfs --cov-report=xml
       env:
         MINIO_ROOT_USER: minioadmin
-        MINIO_ROOT_PASSWORD: minioadmin 
\ No newline at end of file
+        MINIO_ROOT_PASSWORD: minioadmin
+
+    - name: Stop MinIO
+      if: always()
+      run: docker compose -f tests/docker/docker-compose.yml down 

From 6df1af3421ce1d32307a6c0f31ae41b16c3887d6 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:16:37 +0800
Subject: [PATCH 03/18] fix ci to run pyo3 build within venv

---
 .github/workflows/test.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 34d383d..7fea08f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,18 +24,22 @@ jobs:
         # Wait for MinIO to be ready
         sleep 10
 
-    - name: Install dependencies
+    - name: Create and activate virtual environment
       run: |
+        python -m venv .venv
+        source .venv/bin/activate
         python -m pip install --upgrade pip
         pip install maturin pytest pytest-asyncio boto3
 
     - name: Build and install
       run: |
+        source .venv/bin/activate
         maturin develop -m crates/service-memory/Cargo.toml
         maturin develop -m crates/service-s3/Cargo.toml
 
     - name: Run tests
       run: |
+        source .venv/bin/activate
         pytest -v --cov=opendalfs --cov-report=xml
       env:
         MINIO_ROOT_USER: minioadmin

From 328ecd0634eabea8220f619483d1e0cab86e6778 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:31:20 +0800
Subject: [PATCH 04/18] polish pyproject.toml

TODO: leverage PDM for modern package mgmt
---
 .github/workflows/ci.yml   |  79 ++++++++--------
 .github/workflows/test.yml |   5 +-
 .gitignore                 |   2 +
 CONTRIBUTING.md            | 187 ++++++++++++++++++-------------------
 README.md                  |  51 +++++++++-
 pyproject.toml             |  36 ++++++-
 6 files changed, 217 insertions(+), 143 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c731770..6b328a8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,50 +2,47 @@ name: CI
 
 on:
   push:
-    branches:
-      - main
+    branches: [ main ]
   pull_request:
-    branches:
-      - main
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
+    branches: [ main ]
 
 jobs:
-  check-rust:
+  build:
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check format
-        run: cargo fmt --all -- --check
-      - name: Check clippy
-        run: cargo clippy --all-targets --all-features -- -D warnings
 
-  check-python:
-    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Install tools
-        run: |
-          pip install ruff
-      - name: Check format
-        run: |
-          ruff format . --diff
-      - name: Check style
-        run: |
-          ruff check .
-
-  check-pyo3:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        service:
-          - "memory"
-          - "s3"
-    steps:
-      - uses: actions/checkout@v4
-      - uses: PyO3/maturin-action@v1
-        with:
-          command: build
-          working-directory: "crates/service-${{ matrix.service }}"
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Create and activate virtual environment
+      run: |
+        python -m venv .venv
+        source .venv/bin/activate
+        python -m pip install --upgrade pip
+        # Install all dependencies
+        pip install -e ".[all]"
+
+    - name: Check code format
+      run: |
+        source .venv/bin/activate
+        ruff format . --check
+
+    - name: Build service packages
+      run: |
+        source .venv/bin/activate
+        maturin build -m crates/service-memory/Cargo.toml
+        maturin build -m crates/service-s3/Cargo.toml
+
+    - name: Build Python package
+      run: |
+        source .venv/bin/activate
+        python -m build
+
+    - name: Check dist files
+      run: |
+        source .venv/bin/activate
+        twine check dist/*
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7fea08f..ba1597e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,9 +29,10 @@ jobs:
         python -m venv .venv
         source .venv/bin/activate
         python -m pip install --upgrade pip
-        pip install maturin pytest pytest-asyncio boto3
+        # Install package with test and dev dependencies
+        pip install -e ".[test,dev]"
 
-    - name: Build and install
+    - name: Build and install service packages
       run: |
         source .venv/bin/activate
         maturin develop -m crates/service-memory/Cargo.toml
diff --git a/.gitignore b/.gitignore
index f45df55..6ceb951 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,5 @@ venv/
 *.pyc
 .pdm-python
 .idea
+
+opendalfs.egg-info/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6412d14..fee735f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,8 +1,8 @@
 # Contributing
 
-opendalfs is a hybrid of Rust and Python. The underlying implementation is written in Rust, with a Python interface provided by python.
+opendalfs is a hybrid of Rust and Python. The underlying implementation is written in Rust, with a Python interface provided by PyO3.
 
-The overall structure of the project is as follows:
+## Project Structure
 
 ```mermaid
 graph TD;
@@ -17,9 +17,97 @@ graph TD;
     opendalfs-core -- use -->opendal["Apache OpenDAL"];
 ```
 
-- `opendalfs` is the python interface which will implement `fsspec` API and interact with services.
-- `crates/core` is the Rust core implementation.
-- `crates/service-*` provides the service configurations and python packages that used by `opendalfs`.
+- `opendalfs` is the python interface implementing the `fsspec` API
+- `crates/core` is the Rust core implementation
+- `crates/service-*` provides service configurations and Python packages
+
+## Development Setup
+
+1. Clone the repository:
+```shell
+git clone https://github.com/fsspec/opendalfs.git
+cd opendalfs
+```
+
+2. Create a virtual environment and install dependencies:
+```shell
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[all]"  # Install all dependencies
+```
+
+### Dependency Groups
+
+The project uses several dependency groups:
+- `memory`: Memory backend service
+- `s3`: S3 backend service
+- `dev`: Development tools (ruff, maturin, build, twine)
+- `test`: Testing tools (pytest, pytest-asyncio, pytest-cov, fsspec, s3fs)
+- `all`: All dependencies combined
+
+Install specific groups as needed:
+```shell
+pip install -e ".[dev,test]"  # For development and testing
+pip install -e ".[s3]"        # For S3 backend development
+```
+
+## Testing
+
+### Prerequisites
+
+1. For S3 tests, you need MinIO running locally:
+```shell
+docker compose -f tests/docker/docker-compose.yml up -d
+```
+
+Note: The S3 tests use these default settings:
+- Endpoint: http://localhost:9000
+- Region: us-east-1
+- Access Key: minioadmin
+- Secret Key: minioadmin
+- Bucket: test-bucket
+
+### Running Tests
+
+1. Run the test suite:
+```shell
+pytest -v
+```
+
+2. After testing, stop MinIO:
+```shell
+docker compose -f tests/docker/docker-compose.yml down
+```
+
+## Code Style
+
+### Rust
+- Format: `cargo fmt`
+- Lint: `cargo clippy --all-targets --workspace`
+
+### Python
+- Format and lint: `ruff format .`
+- Check: `ruff check .`
+
+## Building
+
+We use [maturin](https://pyo3.rs/) to build Rust extensions for Python:
+
+```shell
+# Build service packages
+maturin develop -m crates/service-memory/Cargo.toml
+maturin develop -m crates/service-s3/Cargo.toml
+```
+
+## CI/CD
+
+Our GitHub Actions workflows handle:
+- Code formatting and linting
+- Building packages
+- Running tests with MinIO
+- Package distribution checks
+
+See `.github/workflows/` for detailed configurations.
 
 ## Async/Sync Architecture
 
@@ -110,92 +198,3 @@ This architecture ensures:
 - Compatibility with FSSpec's patterns
 - Consistent behavior across storage backends
 - Efficient IO handling through dedicated event loops
-
-## Rust
-
-The whole project is a rust workspace, most operations can handled by `cargo`.
-
-- format: `cargo fmt`
-- check: `cargo clippy --all-targets --workspace`
-
-## Python
-
-Use `ruff` to check and format python code:
-
-- format: `ruff format .`
-- check: `ruff check .`
-
-## Testing
-
-Tests are organized into two main categories:
-
-### Core Tests
-Located in `tests/core/`, these test the fundamental functionality that should work across all backends:
-- `test_basic.py`: Basic filesystem operations (create, exists)
-- `test_io.py`: IO operations (read, write)
-- `test_metadata.py`: Metadata operations (info, timestamps)
-- `test_directory.py`: Directory operations (mkdir, ls, rm)
-
-### Backend Tests
-Located in `tests/backends/`, these test backend-specific behaviors:
-- `test_memory.py`: Memory backend specific features
-- `test_s3.py`: S3 backend specific features
-
-### Adding New Tests
-When adding new functionality:
-1. Add core functionality tests to appropriate file in `tests/core/`
-2. If the feature has backend-specific behavior, add tests to `tests/backends/`
-3. Use the existing fixtures `memory_fs` and `s3_fs` from `conftest.py`
-4. Follow the pattern of testing both backends in core tests where applicable
-
-### Prerequisites
-
-1. For S3 tests, you need MinIO running locally. Start it using:
-```shell
-docker compose -f tests/docker/docker-compose.yml up -d
-```
-
-+ Note: The S3 tests use the following default configuration:
-+ - Endpoint: http://localhost:9000
-+ - Region: us-east-1
-+ - Access Key: minioadmin
-+ - Secret Key: minioadmin
-+ - Bucket: test-bucket
-
-### Running Tests
-
-1. Run the test suite:
-```shell
-pytest -v
-```
-
-2. After testing, you can stop MinIO:
-```shell
-docker compose -f tests/docker/docker-compose.yml down
-```
-
-### CI Testing
-
-Our GitHub Actions workflow automatically sets up MinIO for testing in CI. The workflow:
-1. Starts a MinIO container
-2. Builds and installs the packages
-3. Runs the test suite
-
-You can check `.github/workflows/test.yml` for the detailed CI configuration.
-
-## PyO3
-
-We are using [pyo3](https://pyo3.rs/) to call rust from python. Please install `maturin` to build the python package.
-
-```shell
-python -m venv venv
-source venv/bin/activate
-pip install maturin
-```
-
-Build service packages:
-
-```shell
-maturin develop -m crates/service-s3/Cargo.toml
-maturin develop -m crates/service-memory/Cargo.toml
-```
diff --git a/README.md b/README.md
index 886ec19..5f3edc1 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,45 @@
 
 `opendalfs` is a Python-based interface for file systems that enables interaction with different storage services by [Apache OpenDAL](https://github.com/apache/opendal). Through `opendalfs`, users can utilize fsspec's standard API to operate on all [storage services supported by OpenDAL](https://docs.rs/opendal/latest/opendal/services/index.html).
 
+## Installation
+
+### Basic Installation
+```bash
+pip install opendalfs
+```
+
+### With Optional Services
+```bash
+# Install with S3 support
+pip install "opendalfs[s3]"
+
+# Install with memory backend
+pip install "opendalfs[memory]"
+
+# Install with all service backends
+pip install "opendalfs[memory,s3]"
+```
+
+### Development Installation
+```bash
+# Install all development dependencies
+pip install "opendalfs[all]"
+
+# Or install specific groups
+pip install "opendalfs[dev,test]"  # for development and testing
+```
+
+## Development Setup
+
+This project uses:
+- Python 3.11+ for the Python interface
+- Rust for the core implementation
+- maturin for building Rust extensions
+- ruff for code formatting and linting
+- pytest for testing
+
+For development setup and guidelines, see our [Contributing Guide](CONTRIBUTING.md).
+
 ## Status
 
 See [Tracking issues of 0.1.0 version for opendalfs](https://github.com/fsspec/opendalfs/issues/6)
@@ -10,13 +49,15 @@ See [Tracking issues of 0.1.0 version for opendalfs](https://github.com/fsspec/o
 
 opendalfs is an exciting project currently under active development. Whether you're looking to use it in your projects or contribute to its growth, there are several ways you can get involved:
 
-- Follow the [Contributing Guide](CONTRIBUTING.md) to contribute.
-- Create new [Issue](https://github.com/fsspec/opendalfs/issues/new) for bug reporter or feature request.
+- Follow the [Contributing Guide](CONTRIBUTING.md) to contribute
+- Create new [Issue](https://github.com/fsspec/opendalfs/issues/new) for bug reports or feature requests
+- Join discussions in [Discussions](https://github.com/fsspec/opendalfs/discussions)
 
-## Getting help
+## Getting Help
 
-Submit [issues](https://github.com/fsspec/opendalfs/issues/new/choose) for bug report or asking questions in [discussion](https://github.com/fsspec/opendalfs/discussions/new?category=q-a).
+- Submit [issues](https://github.com/fsspec/opendalfs/issues/new/choose) for bug reports
+- Ask questions in [discussions](https://github.com/fsspec/opendalfs/discussions/new?category=q-a)
 
 ## License
 
-Licensed under <a href="./LICENSE">Apache License, Version 2.0</a>.
+Licensed under [Apache License, Version 2.0](./LICENSE).
diff --git a/pyproject.toml b/pyproject.toml
index b8c309f..d55b692 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,40 @@ classifiers = [
 ]
 
 [project.optional-dependencies]
+# Runtime optional dependencies
 memory = ["opendalfs_service_memory"]
 s3 = ["opendalfs_service_s3"]
-test = ["pytest"]
+
+# Development dependencies
+dev = [
+    "ruff",              # Code formatting and linting
+    "maturin",           # Build Rust extensions
+    "build",             # Build Python packages
+    "twine",            # Upload packages to PyPI
+]
+
+# Test dependencies
+test = [
+    "pytest",           # Test framework
+    "pytest-asyncio",   # Async test support
+    "pytest-cov",       # Coverage reporting
+    "fsspec",          # Filesystem interface
+    "s3fs",            # S3 filesystem support
+]
+
+# All development dependencies
+all = ["opendalfs[memory,s3,dev,test]"]
+
+[tool.setuptools]
+packages = ["opendalfs"]
+package-dir = {"" = "."}
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.package-data]
+opendalfs = ["py.typed"]
+
+[tool.setuptools.exclude-package-data]
+"*" = ["crates/*", "target/*", "tests/*"]

From 0bd1ee229002fc3f38d2ac5bfda75a06f99de86e Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:35:19 +0800
Subject: [PATCH 05/18] add missing dep for test

---
 pyproject.toml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d55b692..c4205a8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,7 +20,7 @@ dev = [
     "ruff",              # Code formatting and linting
     "maturin",           # Build Rust extensions
     "build",             # Build Python packages
-    "twine",            # Upload packages to PyPI
+    "twine",             # Upload packages to PyPI
 ]
 
 # Test dependencies
@@ -28,8 +28,9 @@ test = [
     "pytest",           # Test framework
     "pytest-asyncio",   # Async test support
     "pytest-cov",       # Coverage reporting
-    "fsspec",          # Filesystem interface
-    "s3fs",            # S3 filesystem support
+    "fsspec",           # Filesystem interface
+    "s3fs",             # S3 filesystem support
+    "boto3",            # AWS S3 client for environment provision
 ]
 
 # All development dependencies

From cc0555664bfe59342d6db496ecd9b677de451762 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:41:25 +0800
Subject: [PATCH 06/18] fix build with setuptools

---
 .github/workflows/ci.yml   | 23 ++++++++++++++++-------
 .github/workflows/test.yml | 15 +++++++++++----
 .gitignore                 |  1 +
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6b328a8..dc94ab1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,12 +18,25 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Create and activate virtual environment
+    - name: Create virtual environment
       run: |
         python -m venv .venv
         source .venv/bin/activate
         python -m pip install --upgrade pip
-        # Install all dependencies
+        # Install build dependencies first
+        pip install maturin build twine ruff pytest pytest-cov
+
+    - name: Build service packages
+      run: |
+        source .venv/bin/activate
+        # Build and install service packages
+        maturin develop -m crates/service-memory/Cargo.toml
+        maturin develop -m crates/service-s3/Cargo.toml
+
+    - name: Install main package
+      run: |
+        source .venv/bin/activate
+        # Now install the main package with all extras
         pip install -e ".[all]"
 
     - name: Check code format
@@ -31,15 +44,11 @@ jobs:
         source .venv/bin/activate
         ruff format . --check
 
-    - name: Build service packages
+    - name: Build packages for distribution
       run: |
         source .venv/bin/activate
         maturin build -m crates/service-memory/Cargo.toml
         maturin build -m crates/service-s3/Cargo.toml
-
-    - name: Build Python package
-      run: |
-        source .venv/bin/activate
         python -m build
 
     - name: Check dist files
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ba1597e..6c45aa3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,20 +24,27 @@ jobs:
         # Wait for MinIO to be ready
         sleep 10
 
-    - name: Create and activate virtual environment
+    - name: Create virtual environment
       run: |
         python -m venv .venv
         source .venv/bin/activate
         python -m pip install --upgrade pip
-        # Install package with test and dev dependencies
-        pip install -e ".[test,dev]"
+        # Install build dependencies first
+        pip install maturin pytest pytest-cov
 
-    - name: Build and install service packages
+    - name: Build service packages
       run: |
         source .venv/bin/activate
+        # Build and install service packages
         maturin develop -m crates/service-memory/Cargo.toml
         maturin develop -m crates/service-s3/Cargo.toml
 
+    - name: Install main package
+      run: |
+        source .venv/bin/activate
+        # Now install the main package with test extras
+        pip install -e ".[test]"
+
     - name: Run tests
       run: |
         source .venv/bin/activate
diff --git a/.gitignore b/.gitignore
index 6ceb951..7f9fb44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ venv/
 .idea
 
 opendalfs.egg-info/
+dist/

From 8f9e64c5e1a507d2b4febd29fcb08fc81cd7f19f Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 14:53:09 +0800
Subject: [PATCH 07/18] refine ci workflow

---
 .github/workflows/ci.yml                  | 57 -----------------------
 .github/workflows/package.yml             | 50 ++++++++++++++++++++
 .github/workflows/{test.yml => tests.yml} | 35 +++++++-------
 3 files changed, 67 insertions(+), 75 deletions(-)
 delete mode 100644 .github/workflows/ci.yml
 create mode 100644 .github/workflows/package.yml
 rename .github/workflows/{test.yml => tests.yml} (66%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index dc94ab1..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-
-    - name: Create virtual environment
-      run: |
-        python -m venv .venv
-        source .venv/bin/activate
-        python -m pip install --upgrade pip
-        # Install build dependencies first
-        pip install maturin build twine ruff pytest pytest-cov
-
-    - name: Build service packages
-      run: |
-        source .venv/bin/activate
-        # Build and install service packages
-        maturin develop -m crates/service-memory/Cargo.toml
-        maturin develop -m crates/service-s3/Cargo.toml
-
-    - name: Install main package
-      run: |
-        source .venv/bin/activate
-        # Now install the main package with all extras
-        pip install -e ".[all]"
-
-    - name: Check code format
-      run: |
-        source .venv/bin/activate
-        ruff format . --check
-
-    - name: Build packages for distribution
-      run: |
-        source .venv/bin/activate
-        maturin build -m crates/service-memory/Cargo.toml
-        maturin build -m crates/service-s3/Cargo.toml
-        python -m build
-
-    - name: Check dist files
-      run: |
-        source .venv/bin/activate
-        twine check dist/*
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
new file mode 100644
index 0000000..bbadc4d
--- /dev/null
+++ b/.github/workflows/package.yml
@@ -0,0 +1,50 @@
+name: Code Checks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  verify:
+    name: Lint & Build
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Set up development environment
+      run: |
+        python -m venv .venv
+        source .venv/bin/activate
+        python -m pip install --upgrade pip
+        pip install maturin build twine ruff
+
+    - name: Check code formatting
+      run: |
+        source .venv/bin/activate
+        ruff format . --check
+
+    - name: Build Rust extensions
+      run: |
+        source .venv/bin/activate
+        # Build service backends
+        maturin build -m crates/service-memory/Cargo.toml
+        maturin build -m crates/service-s3/Cargo.toml
+
+    - name: Build Python package
+      run: |
+        source .venv/bin/activate
+        python -m build
+
+    - name: Verify packages
+      run: |
+        source .venv/bin/activate
+        twine check dist/* 
\ No newline at end of file
diff --git a/.github/workflows/test.yml b/.github/workflows/tests.yml
similarity index 66%
rename from .github/workflows/test.yml
rename to .github/workflows/tests.yml
index 6c45aa3..42ee1f4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/tests.yml
@@ -1,4 +1,4 @@
-name: Test
+name: Tests
 
 on:
   push:
@@ -8,51 +8,50 @@ on:
 
 jobs:
   test:
+    name: Integration Tests
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-    - name: Set up Python
+    - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
         python-version: '3.11'
 
-    - name: Start MinIO
+    - name: Start MinIO server
       run: |
         docker compose -f tests/docker/docker-compose.yml up -d
-        # Wait for MinIO to be ready
+        # Wait for MinIO to be healthy
         sleep 10
 
-    - name: Create virtual environment
+    - name: Set up test environment
       run: |
         python -m venv .venv
         source .venv/bin/activate
         python -m pip install --upgrade pip
-        # Install build dependencies first
-        pip install maturin pytest pytest-cov
+        pip install maturin pytest pytest-cov pytest-asyncio
 
-    - name: Build service packages
+    - name: Install service backends
       run: |
         source .venv/bin/activate
-        # Build and install service packages
         maturin develop -m crates/service-memory/Cargo.toml
         maturin develop -m crates/service-s3/Cargo.toml
 
-    - name: Install main package
+    - name: Install package
       run: |
         source .venv/bin/activate
-        # Now install the main package with test extras
         pip install -e ".[test]"
 
-    - name: Run tests
-      run: |
-        source .venv/bin/activate
-        pytest -v --cov=opendalfs --cov-report=xml
+    - name: Run test suite
       env:
         MINIO_ROOT_USER: minioadmin
         MINIO_ROOT_PASSWORD: minioadmin
+      run: |
+        source .venv/bin/activate
+        pytest -v --cov=opendalfs --cov-report=xml
 
-    - name: Stop MinIO
+    - name: Cleanup MinIO server
       if: always()
-      run: docker compose -f tests/docker/docker-compose.yml down 
+      run: docker compose -f tests/docker/docker-compose.yml down 
\ No newline at end of file

From 5011f6658c3d0f3c67ee5385854132fc28005e45 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:01:43 +0800
Subject: [PATCH 08/18] chore: badges of ci and discussions

---
 .github/workflows/package.yml | 2 +-
 .github/workflows/tests.yml   | 8 ++++++++
 README.md                     | 5 +++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index bbadc4d..f2a2b7e 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -1,4 +1,4 @@
-name: Code Checks
+name: Package Build & Lint
 
 on:
   push:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 42ee1f4..4dd96e8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -3,8 +3,16 @@ name: Tests
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
   pull_request:
     branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
 
 jobs:
   test:
diff --git a/README.md b/README.md
index 5f3edc1..91e16d4 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,10 @@
 # opendalfs
 
+[![Discussions](https://img.shields.io/github/discussions/fsspec/opendalfs)](https://github.com/fsspec/opendalfs/discussions)
+[![Tests](https://github.com/fsspec/opendalfs/actions/workflows/tests.yml/badge.svg)](https://github.com/fsspec/opendalfs/actions/workflows/tests.yml)
+[![Package](https://github.com/fsspec/opendalfs/actions/workflows/package.yml/badge.svg)](https://github.com/fsspec/opendalfs/actions/workflows/package.yml)
+
+
 `opendalfs` is a Python-based interface for file systems that enables interaction with different storage services by [Apache OpenDAL](https://github.com/apache/opendal). Through `opendalfs`, users can utilize fsspec's standard API to operate on all [storage services supported by OpenDAL](https://docs.rs/opendal/latest/opendal/services/index.html).
 
 ## Installation

From 991ae5676024a3ea8db0f8a69dd2a046bb9a1f87 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:08:31 +0800
Subject: [PATCH 09/18] refine package.yml

---
 .github/workflows/package.yml | 124 +++++++++++++++++++++++-----------
 1 file changed, 85 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index f2a2b7e..aa6c9f2 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -1,50 +1,96 @@
-name: Package Build & Lint
+name: Package
 
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
   pull_request:
     branches: [ main ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.rst'
+      - 'docs/**'
 
 jobs:
-  verify:
-    name: Lint & Build
+  check-rust:
+    name: Rust Checks
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check format
+        run: cargo fmt --all -- --check
+
+      - name: Check clippy
+        run: cargo clippy --all-targets --all-features -- -D warnings
 
+  check-python:
+    name: Python Checks
+    runs-on: ubuntu-latest
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-
-    - name: Set up development environment
-      run: |
-        python -m venv .venv
-        source .venv/bin/activate
-        python -m pip install --upgrade pip
-        pip install maturin build twine ruff
-
-    - name: Check code formatting
-      run: |
-        source .venv/bin/activate
-        ruff format . --check
-
-    - name: Build Rust extensions
-      run: |
-        source .venv/bin/activate
-        # Build service backends
-        maturin build -m crates/service-memory/Cargo.toml
-        maturin build -m crates/service-s3/Cargo.toml
-
-    - name: Build Python package
-      run: |
-        source .venv/bin/activate
-        python -m build
-
-    - name: Verify packages
-      run: |
-        source .venv/bin/activate
-        twine check dist/* 
\ No newline at end of file
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install tools
+        run: pip install ruff build twine
+
+      - name: Check format
+        run: ruff format . --check
+
+      - name: Check style
+        run: ruff check .
+
+  check-pyo3:
+    name: PyO3 Service Builds
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        service: ["memory", "s3"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          working-directory: "crates/service-${{ matrix.service }}"
+
+  build-dist:
+    name: Build Distribution
+    needs: [check-rust, check-python, check-pyo3]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build Python package
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+          python -m build
+
+      - name: Verify package
+        run: |
+          pip install twine
+          twine check dist/*
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error 
\ No newline at end of file

From 2b6e0e9eb9778021b8cd0e7ea121d95aa4ac0e03 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:21:46 +0800
Subject: [PATCH 10/18] better build and packaging ci

---
 .github/workflows/package.yml | 126 ++++++++++++++++++++++++++++------
 1 file changed, 104 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index aa6c9f2..b738a3f 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -49,24 +49,72 @@ jobs:
       - name: Check style
         run: ruff check .
 
-  check-pyo3:
-    name: PyO3 Service Builds
-    runs-on: ubuntu-latest
+  build-packages:
+    name: Build Packages
+    needs: [check-rust, check-python]
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        service: ["memory", "s3"]
+        os: [ubuntu-latest, macos-latest]
+        target: [x86_64, aarch64]
+        service: [memory, s3]
+        exclude:
+          # Exclude aarch64 builds on x86 runners
+          - os: ubuntu-latest
+            target: aarch64
+          - os: macos-latest
+            target: aarch64
+        include:
+          # Linux builds use manylinux
+          - os: ubuntu-latest
+            target: x86_64
+            platform: manylinux_2_28_x86_64
+          # macOS builds are native
+          - os: macos-latest
+            target: x86_64
+            platform: native
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - uses: PyO3/maturin-action@v1
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build service package
+        uses: PyO3/maturin-action@v1
         with:
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.platform }}
+          args: --release --out dist
           command: build
-          working-directory: "crates/service-${{ matrix.service }}"
+          working-directory: crates/service-${{ matrix.service }}
 
-  build-dist:
-    name: Build Distribution
-    needs: [check-rust, check-python, check-pyo3]
+      # Build main package only once per OS
+      - name: Build main package
+        if: matrix.service == 'memory'  # Only build once per OS
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build --outdir dist/
+
+      - name: Verify packages
+        run: |
+          pip install twine
+          twine check dist/*
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.service }}
+          path: dist/
+          if-no-files-found: error
+
+  verify-install:
+    name: Verify Installation
+    needs: build-packages
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -77,20 +125,54 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Build Python package
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: dist
+
+      - name: Prepare dist directory
         run: |
-          python -m pip install --upgrade pip
-          pip install build
-          python -m build
+          mkdir -p dist_combined
+          cp -r dist/*/* dist_combined/
 
-      - name: Verify package
+      - name: Install packages
         run: |
-          pip install twine
-          twine check dist/*
+          python -m pip install --upgrade pip
+          pip install dist_combined/*.whl
+          pip install "opendalfs[all]"
 
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist/
-          if-no-files-found: error 
\ No newline at end of file
+      - name: Verify imports
+        run: |
+          python -c "import opendalfs"
+          python -c "import opendalfs_service_memory"
+          python -c "import opendalfs_service_s3"
+
+  # PyPI publishing job (commented out for future use)
+  # publish:
+  #   name: Publish to PyPI
+  #   needs: verify-install
+  #   runs-on: ubuntu-latest
+  #   # Only run on tags
+  #   if: startsWith(github.ref, 'refs/tags/')
+  #   
+  #   steps:
+  #     - name: Download all artifacts
+  #       uses: actions/download-artifact@v3
+  #       with:
+  #         path: dist
+  #
+  #     - name: Prepare dist directory
+  #       run: |
+  #         mkdir -p dist_combined
+  #         cp -r dist/*/* dist_combined/
+  #
+  #     - name: Publish to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+  #       with:
+  #         password: ${{ secrets.PYPI_API_TOKEN }}
+  #         packages-dir: dist_combined/
+  #
+  # Required secrets for PyPI publishing:
+  # - PYPI_API_TOKEN: API token from PyPI
+  #   Get it from: https://pypi.org/manage/account/token/
+  #   Add it to: https://github.com/fsspec/opendalfs/settings/secrets/actions 
\ No newline at end of file

From 3133acbd0859734d1a6b20b5ec8a9a16692343c4 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:28:08 +0800
Subject: [PATCH 11/18] use manylinux container for pyo3 build

---
 .github/workflows/package.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index b738a3f..40163c5 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -17,7 +17,7 @@ on:
 jobs:
   check-rust:
     name: Rust Checks
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -30,7 +30,7 @@ jobs:
 
   check-python:
     name: Python Checks
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -56,20 +56,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-24.04, macos-latest]
         target: [x86_64, aarch64]
         service: [memory, s3]
         exclude:
           # Exclude aarch64 builds on x86 runners
-          - os: ubuntu-latest
+          - os: ubuntu-24.04
             target: aarch64
           - os: macos-latest
             target: aarch64
         include:
           # Linux builds use manylinux
-          - os: ubuntu-latest
+          - os: ubuntu-24.04
             target: x86_64
-            platform: manylinux_2_28_x86_64
+            platform: manylinux2014
           # macOS builds are native
           - os: macos-latest
             target: x86_64
@@ -89,6 +89,7 @@ jobs:
         with:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.platform }}
+          container: quay.io/pypa/manylinux2014_x86_64  # Specify manylinux container
           args: --release --out dist
           command: build
           working-directory: crates/service-${{ matrix.service }}
@@ -115,7 +116,7 @@ jobs:
   verify-install:
     name: Verify Installation
     needs: build-packages
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -151,7 +152,7 @@ jobs:
   # publish:
   #   name: Publish to PyPI
   #   needs: verify-install
-  #   runs-on: ubuntu-latest
+  #   runs-on: ubuntu-24.04
   #   # Only run on tags
   #   if: startsWith(github.ref, 'refs/tags/')
   #   

From 8b3237aece06c7944cb650afac980fda98443bf8 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:39:38 +0800
Subject: [PATCH 12/18] ensure verification of built artifacts

---
 .github/workflows/package.yml | 58 +++++++++++++++++++++++++++--------
 .github/workflows/tests.yml   | 12 ++++----
 2 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 40163c5..30efa04 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -60,17 +60,14 @@ jobs:
         target: [x86_64, aarch64]
         service: [memory, s3]
         exclude:
-          # Exclude aarch64 builds on x86 runners
           - os: ubuntu-24.04
             target: aarch64
           - os: macos-latest
             target: aarch64
         include:
-          # Linux builds use manylinux
           - os: ubuntu-24.04
             target: x86_64
             platform: manylinux2014
-          # macOS builds are native
           - os: macos-latest
             target: x86_64
             platform: native
@@ -84,24 +81,31 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Create dist directory
+        run: mkdir -p dist
+
       - name: Build service package
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.platform }}
-          container: quay.io/pypa/manylinux2014_x86_64  # Specify manylinux container
+          container: quay.io/pypa/manylinux2014_x86_64
           args: --release --out dist
           command: build
           working-directory: crates/service-${{ matrix.service }}
 
-      # Build main package only once per OS
+      # Build main package only once per OS/target combination
       - name: Build main package
-        if: matrix.service == 'memory'  # Only build once per OS
+        if: matrix.service == 'memory'  # Only build once per OS/target
         run: |
           python -m pip install --upgrade pip build
           python -m build --outdir dist/
 
+      - name: List dist contents
+        run: ls -la dist/
+
       - name: Verify packages
+        if: hashFiles('dist/*.whl') != ''
         run: |
           pip install twine
           twine check dist/*
@@ -117,30 +121,47 @@ jobs:
     name: Verify Installation
     needs: build-packages
     runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ['3.11']
+        os: [ubuntu-24.04]  # We test installation on Linux
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.11
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python-version }}
 
       - name: Download all artifacts
         uses: actions/download-artifact@v3
         with:
           path: dist
 
+      - name: List downloaded artifacts
+        run: |
+          find dist -type f
+
       - name: Prepare dist directory
         run: |
           mkdir -p dist_combined
-          cp -r dist/*/* dist_combined/
-
-      - name: Install packages
+          # First copy service packages
+          cp -r dist/dist-${{ matrix.os }}-x86_64-memory/* dist_combined/
+          cp -r dist/dist-${{ matrix.os }}-x86_64-s3/* dist_combined/
+          # Then copy main package
+          cp -r dist/dist-${{ matrix.os }}-x86_64-memory/*.tar.gz dist_combined/ || true
+          ls -la dist_combined/
+
+      - name: Install packages in correct order
         run: |
           python -m pip install --upgrade pip
-          pip install dist_combined/*.whl
-          pip install "opendalfs[all]"
+          # First install service packages
+          pip install dist_combined/*service_memory*.whl
+          pip install dist_combined/*service_s3*.whl
+          # Then install main package
+          pip install dist_combined/opendalfs*.whl
 
       - name: Verify imports
         run: |
@@ -148,6 +169,17 @@ jobs:
           python -c "import opendalfs_service_memory"
           python -c "import opendalfs_service_s3"
 
+      - name: Test full installation
+        run: |
+          # Clean previous installation
+          pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3
+          # Install with all extras
+          pip install dist_combined/opendalfs*.whl[all]
+          # Verify again
+          python -c "import opendalfs"
+          python -c "import opendalfs_service_memory"
+          python -c "import opendalfs_service_s3"
+
   # PyPI publishing job (commented out for future use)
   # publish:
   #   name: Publish to PyPI
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4dd96e8..c999e47 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -15,13 +15,13 @@ on:
       - 'docs/**'
 
 jobs:
-  test:
+  integration-tests:
     name: Integration Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python 3.11
       uses: actions/setup-python@v4
@@ -34,20 +34,20 @@ jobs:
         # Wait for MinIO to be healthy
         sleep 10
 
-    - name: Set up test environment
+    - name: Set up development environment
       run: |
         python -m venv .venv
         source .venv/bin/activate
         python -m pip install --upgrade pip
         pip install maturin pytest pytest-cov pytest-asyncio
 
-    - name: Install service backends
+    - name: Build Rust extensions
       run: |
         source .venv/bin/activate
         maturin develop -m crates/service-memory/Cargo.toml
         maturin develop -m crates/service-s3/Cargo.toml
 
-    - name: Install package
+    - name: Install package with test dependencies
       run: |
         source .venv/bin/activate
         pip install -e ".[test]"

From 96937f641d49fe14a236b264cbd07a06da512517 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 15:51:03 +0800
Subject: [PATCH 13/18] fix main and service build tests

---
 .github/workflows/package.yml | 70 ++++++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 30efa04..10dd535 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -49,10 +49,41 @@ jobs:
       - name: Check style
         run: ruff check .
 
-  build-packages:
-    name: Build Packages
+  build-main:
+    name: Build Main Package
     needs: [check-rust, check-python]
     runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Create dist directory
+        run: mkdir -p dist
+
+      - name: Build main package
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build --outdir dist/
+
+      - name: List dist contents
+        run: ls -la dist/
+
+      - name: Upload main artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist-main
+          path: dist/*
+          if-no-files-found: error
+
+  build-services:
+    name: Build Service Packages
+    needs: [build-main]
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
@@ -84,7 +115,14 @@ jobs:
       - name: Create dist directory
         run: mkdir -p dist
 
+      - name: Download main package
+        uses: actions/download-artifact@v3
+        with:
+          name: dist-main
+          path: dist/
+
       - name: Build service package
+        id: build-service
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
@@ -94,46 +132,28 @@ jobs:
           command: build
           working-directory: crates/service-${{ matrix.service }}
 
-      # Build main package only once per OS/target combination
-      - name: Build main package
-        if: matrix.service == 'memory'  # Only build once per OS/target
-        run: |
-          python -m pip install --upgrade pip build
-          python -m build --outdir dist/
-
       - name: List dist contents
         run: ls -la dist/
 
-      - name: Verify packages
-        if: hashFiles('dist/*.whl') != ''
-        run: |
-          pip install twine
-          twine check dist/*
-
-      - name: Upload artifacts
+      - name: Upload service artifact
         uses: actions/upload-artifact@v3
         with:
           name: dist-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.service }}
-          path: dist/
+          path: dist/*.whl
           if-no-files-found: error
 
   verify-install:
     name: Verify Installation
-    needs: build-packages
+    needs: [build-services]
     runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        python-version: ['3.11']
-        os: [ubuntu-24.04]  # We test installation on Linux
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
 
       - name: Download all artifacts
         uses: actions/download-artifact@v3

From f8a101658bb03a826d80c67a04989efff1309eac Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 19:14:32 +0800
Subject: [PATCH 14/18] final fixes

---
 .github/workflows/package.yml | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 10dd535..ef7d61d 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -40,8 +40,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Install tools
-        run: pip install ruff build twine
+      - name: Install dev dependencies
+        run: pip install -e ".[dev]"
 
       - name: Check format
         run: ruff format . --check
@@ -52,7 +52,7 @@ jobs:
   build-main:
     name: Build Main Package
     needs: [check-rust, check-python]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -83,26 +83,21 @@ jobs:
   build-services:
     name: Build Service Packages
     needs: [build-main]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-latest]
-        target: [x86_64, aarch64]
+        os: [ubuntu-24.04]
+        target: [x86_64]
         service: [memory, s3]
         exclude:
           - os: ubuntu-24.04
             target: aarch64
-          - os: macos-latest
-            target: aarch64
         include:
           - os: ubuntu-24.04
             target: x86_64
             platform: manylinux2014
-          - os: macos-latest
-            target: x86_64
-            platform: native
-
+  
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -168,10 +163,10 @@ jobs:
         run: |
           mkdir -p dist_combined
           # First copy service packages
-          cp -r dist/dist-${{ matrix.os }}-x86_64-memory/* dist_combined/
-          cp -r dist/dist-${{ matrix.os }}-x86_64-s3/* dist_combined/
+          cp -r dist/dist-ubuntu-24.04-x86_64-memory/* dist_combined/
+          cp -r dist/dist-ubuntu-24.04-x86_64-s3/* dist_combined/
           # Then copy main package
-          cp -r dist/dist-${{ matrix.os }}-x86_64-memory/*.tar.gz dist_combined/ || true
+          cp -r dist/dist-main/* dist_combined/
           ls -la dist_combined/
 
       - name: Install packages in correct order

From 9f88d1f77149f73a02a9b491a94fa309e05e4cfb Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 19:19:59 +0800
Subject: [PATCH 15/18] fix verify install steps... again

---
 .github/workflows/package.yml | 92 +++++++++++++++++++++++++++--------
 pyproject.toml                |  6 ++-
 2 files changed, 76 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index ef7d61d..da4d9dd 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -110,11 +110,17 @@ jobs:
       - name: Create dist directory
         run: mkdir -p dist
 
-      - name: Download main package
-        uses: actions/download-artifact@v3
-        with:
-          name: dist-main
-          path: dist/
+      - name: Install build dependencies
+        run: pip install maturin
+
+      - name: Build service package (debug)
+        run: |
+          cd crates/service-${{ matrix.service }}
+          maturin build --release --out ../../dist
+          echo "=== Build directory contents ==="
+          find ../../dist -type f
+          echo "=== Target wheels contents ==="
+          find ../../target/wheels -type f || true
 
       - name: Build service package
         id: build-service
@@ -123,12 +129,26 @@ jobs:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.platform }}
           container: quay.io/pypa/manylinux2014_x86_64
-          args: --release --out dist
+          args: --release --out dist --interpreter python3.11
           command: build
           working-directory: crates/service-${{ matrix.service }}
 
       - name: List dist contents
-        run: ls -la dist/
+        run: |
+          echo "=== Service build output ==="
+          ls -la dist/
+          echo "=== Wheel details ==="
+          find dist -name "*.whl" -exec sh -c 'echo "$1"; unzip -l "$1"' _ {} \;
+          echo "=== All possible wheel locations ==="
+          find . -name "*.whl"
+
+      - name: Copy wheels to dist
+        run: |
+          # Ensure all wheels are in dist directory
+          mkdir -p dist
+          find . -name "*.whl" -exec cp {} dist/ \;
+          echo "=== Final dist contents ==="
+          ls -la dist/
 
       - name: Upload service artifact
         uses: actions/upload-artifact@v3
@@ -157,29 +177,49 @@ jobs:
 
       - name: List downloaded artifacts
         run: |
+          echo "=== All artifacts ==="
           find dist -type f
+          echo "=== Service artifacts ==="
+          find dist/dist-ubuntu-24.04-x86_64-memory dist/dist-ubuntu-24.04-x86_64-s3 -type f
 
       - name: Prepare dist directory
         run: |
           mkdir -p dist_combined
-          # First copy service packages
-          cp -r dist/dist-ubuntu-24.04-x86_64-memory/* dist_combined/
-          cp -r dist/dist-ubuntu-24.04-x86_64-s3/* dist_combined/
-          # Then copy main package
-          cp -r dist/dist-main/* dist_combined/
+          # Copy service wheels first (only cp311 wheels)
+          cp dist/dist-ubuntu-24.04-x86_64-memory/*cp311*.whl dist_combined/
+          cp dist/dist-ubuntu-24.04-x86_64-s3/*cp311*.whl dist_combined/
+          # Copy main package
+          cp dist/dist-main/*.whl dist_combined/
+          cp dist/dist-main/*.tar.gz dist_combined/
+          echo "=== Contents of dist_combined ==="
           ls -la dist_combined/
 
       - name: Install packages in correct order
         run: |
           python -m pip install --upgrade pip
-          # First install service packages
-          pip install dist_combined/*service_memory*.whl
-          pip install dist_combined/*service_s3*.whl
-          # Then install main package
-          pip install dist_combined/opendalfs*.whl
+          echo "=== Installing service packages ==="
+          # Install memory service (only cp311 wheels)
+          for whl in dist_combined/opendalfs_service_memory*cp311*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing memory service: $whl"
+              pip install "$whl"
+            fi
+          done
+          # Install s3 service (only cp311 wheels)
+          for whl in dist_combined/opendalfs_service_s3*cp311*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing s3 service: $whl"
+              pip install "$whl"
+            fi
+          done
+          echo "=== Installing main package ==="
+          pip install dist_combined/opendalfs-*.whl
 
       - name: Verify imports
         run: |
+          echo "=== Installed packages ==="
+          pip list | grep opendalfs
+          echo "=== Verifying imports ==="
           python -c "import opendalfs"
           python -c "import opendalfs_service_memory"
           python -c "import opendalfs_service_s3"
@@ -188,9 +228,21 @@ jobs:
         run: |
           # Clean previous installation
           pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3
-          # Install with all extras
-          pip install dist_combined/opendalfs*.whl[all]
-          # Verify again
+          # Install with all extras (use exact filename)
+          echo "=== Installing with all extras ==="
+          # First install service packages
+          for whl in dist_combined/opendalfs_service_*manylinux_2_17*.whl; do
+            if [ -f "$whl" ]; then
+              echo "Installing service: $whl"
+              pip install "$whl"
+            fi
+          done
+          # Then install main package with all extras
+          MAIN_WHEEL=$(find dist_combined -name "opendalfs-*.whl")
+          echo "Installing main package with extras: $MAIN_WHEEL"
+          pip install "$MAIN_WHEEL[all]"
+          # Verify imports
+          echo "=== Verifying imports ==="
           python -c "import opendalfs"
           python -c "import opendalfs_service_memory"
           python -c "import opendalfs_service_s3"
diff --git a/pyproject.toml b/pyproject.toml
index c4205a8..4ea3cba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,12 +3,15 @@ name = "opendalfs"
 version = "0.0.1"
 classifiers = [
     "Development Status :: 4 - Beta",
-    "Intended Audience :: Developers",
+    "Intended Audience :: Developers", 
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
+dependencies = [
+    "fsspec",
+]
 
 [project.optional-dependencies]
 # Runtime optional dependencies
@@ -28,7 +31,6 @@ test = [
     "pytest",           # Test framework
     "pytest-asyncio",   # Async test support
     "pytest-cov",       # Coverage reporting
-    "fsspec",           # Filesystem interface
     "s3fs",             # S3 filesystem support
     "boto3",            # AWS S3 client for environment provision
 ]

From 1ede2ca8a27a64d0e371753d66d1c8e7fdab7e0d Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 20:39:43 +0800
Subject: [PATCH 16/18] final polishment

---
 .github/workflows/package.yml | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index da4d9dd..d08abb1 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -1,5 +1,12 @@
 name: Package
 
+# This workflow handles:
+# 1. Code quality checks for both Rust and Python
+# 2. Building the main Python package
+# 3. Building Rust service extensions (memory, s3)
+# 4. Verifying package installation and imports
+# Future: PyPI publishing on tags
+
 on:
   push:
     branches: [ main ]
@@ -18,10 +25,20 @@ jobs:
   check-rust:
     name: Rust Checks
     runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Cache Rust dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
       - name: Check format
         run: cargo fmt --all -- --check
 
@@ -31,10 +48,17 @@ jobs:
   check-python:
     name: Python Checks
     runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+
       - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
@@ -53,6 +77,7 @@ jobs:
     name: Build Main Package
     needs: [check-rust, check-python]
     runs-on: ubuntu-24.04
+    timeout-minutes: 15
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -107,9 +132,6 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Create dist directory
-        run: mkdir -p dist
-
       - name: Install build dependencies
         run: pip install maturin
 
@@ -161,6 +183,7 @@ jobs:
     name: Verify Installation
     needs: [build-services]
     runs-on: ubuntu-24.04
+    timeout-minutes: 15
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -247,6 +270,12 @@ jobs:
           python -c "import opendalfs_service_memory"
           python -c "import opendalfs_service_s3"
 
+      - name: Cleanup
+        if: always()
+        run: |
+          pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3 || true
+          rm -rf dist dist_combined
+
   # PyPI publishing job (commented out for future use)
   # publish:
   #   name: Publish to PyPI

From 5e91cb6bb6f0578a50e3a1a089fc33475cf22cc5 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 21:10:47 +0800
Subject: [PATCH 17/18] fixed pyo3 build whl upload

---
 .github/workflows/package.yml | 31 ++++++-------------------------
 1 file changed, 6 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index d08abb1..e81bdfe 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -135,15 +135,6 @@ jobs:
       - name: Install build dependencies
         run: pip install maturin
 
-      - name: Build service package (debug)
-        run: |
-          cd crates/service-${{ matrix.service }}
-          maturin build --release --out ../../dist
-          echo "=== Build directory contents ==="
-          find ../../dist -type f
-          echo "=== Target wheels contents ==="
-          find ../../target/wheels -type f || true
-
       - name: Build service package
         id: build-service
         uses: PyO3/maturin-action@v1
@@ -155,28 +146,18 @@ jobs:
           command: build
           working-directory: crates/service-${{ matrix.service }}
 
-      - name: List dist contents
-        run: |
-          echo "=== Service build output ==="
-          ls -la dist/
-          echo "=== Wheel details ==="
-          find dist -name "*.whl" -exec sh -c 'echo "$1"; unzip -l "$1"' _ {} \;
-          echo "=== All possible wheel locations ==="
-          find . -name "*.whl"
-
-      - name: Copy wheels to dist
+      - name: List all wheels
         run: |
-          # Ensure all wheels are in dist directory
-          mkdir -p dist
-          find . -name "*.whl" -exec cp {} dist/ \;
-          echo "=== Final dist contents ==="
-          ls -la dist/
+          echo "=== All wheel locations ==="
+          find . -name "*.whl" -exec ls -l {} \;
+          echo "=== Service directory contents ==="
+          ls -la crates/service-${{ matrix.service }}/dist/
 
       - name: Upload service artifact
         uses: actions/upload-artifact@v3
         with:
           name: dist-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.service }}
-          path: dist/*.whl
+          path: crates/service-${{ matrix.service }}/dist/*.whl
           if-no-files-found: error
 
   verify-install:

From 0cb6f18816bc1effc6b8476d61efdc96f1149be7 Mon Sep 17 00:00:00 2001
From: Wey Gu <weyl.gu@gmail.com>
Date: Sun, 5 Jan 2025 21:20:43 +0800
Subject: [PATCH 18/18] chore: update contribution doc

---
 CONTRIBUTING.md | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index fee735f..6362efc 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -56,25 +56,29 @@ pip install -e ".[s3]"        # For S3 backend development
 ### Prerequisites
 
 1. For S3 tests, you need MinIO running locally:
+
 ```shell
 docker compose -f tests/docker/docker-compose.yml up -d
 ```
 
 Note: The S3 tests use these default settings:
-- Endpoint: http://localhost:9000
-- Region: us-east-1
-- Access Key: minioadmin
-- Secret Key: minioadmin
-- Bucket: test-bucket
+
+- Endpoint: `http://localhost:9000`
+- Region: `us-east-1`
+- Access Key: `minioadmin`
+- Secret Key: `minioadmin`
+- Bucket: `test-bucket`
 
 ### Running Tests
 
 1. Run the test suite:
+
 ```shell
 pytest -v
 ```
 
 2. After testing, stop MinIO:
+
 ```shell
 docker compose -f tests/docker/docker-compose.yml down
 ```
@@ -116,7 +120,7 @@ See `.github/workflows/` for detailed configurations.
 We use `pyo3-asyncio` to bridge between Rust futures and Python coroutines:
 
 ```mermaid
-graph TD;
+graph LR;
     Python[Python Sync API] --> FSSpec[FSSpec sync_wrapper];
     FSSpec --> AsyncFS[AsyncFileSystem];
     AsyncFS --> PyO3[PyO3 Async Bridge];