Add support for Linux aarch64 build (#27)

* Adds support for Linux `aarch64` builds * Makes the build script more generic, and shares the code for all the `linux` builds * All the build targets are stored and transported using github actions artifacts * Pushes the packages to `pypi` in the final phase iff all the packages are built successfully. * Add both `manylinux2014` & `manylinux2010` for `x86_64` * Bump the version to `0.4` --------- Co-authored-by: Albert Torosyan <torosyanalbert@gmail.com>
aimhubio · Mar 1, 2023 · 91a9e57 · 91a9e57
1 parent b9774af
commit 91a9e57
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 22 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -3,13 +3,86 @@ name: aimrocks packaging pipeline
 on: workflow_dispatch
 
 jobs:
-  linux-dist:
+  uploading-wheels:
+    needs:
+      - linux-dist-aarch64
+      - linux-dist-x86_64
+      - macos-dist
+    runs-on: ubuntu-latest
+    name: Uploading wheels
+    steps:
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: x64
+
+      - name: Install dev dependencies
+        run: |
+          python -m pip install twine
+          
+      - name: Download all the wheels
+        uses: actions/download-artifact@v3
+
+      - name: Publish wheels
+        env:
+          PYPI_PASSWORD: ${{ secrets.pypi_password }}
+        run: |
+          python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" artifact/*.whl
+
+  linux-dist-aarch64:
+    runs-on: m1
+    strategy:
+      fail-fast: false
+      matrix:
+        manylinux-version: [ 'manylinux_2_28_aarch64', 'manylinux2014_aarch64', 'manylinux_2_24_aarch64' ]
+    name: ${{ matrix.manylinux-version }} using Docker on M1 Mac
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Pull Docker Images
+        run: |
+         docker pull quay.io/pypa/${{ matrix.manylinux-version }}
+
+      - name: Building dependencies for ${{ matrix.manylinux-version }}
+        run: |
+         docker build \
+         --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \
+         --target deps .
+
+      - name: Building rocksdb for ${{ matrix.manylinux-version }}
+        run: |
+         docker build \
+         --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \
+         --target rocksdb .
+
+      - name: Building wheels for ${{ matrix.manylinux-version }}
+        run: |
+         docker build \
+         --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \
+         --target wheels . \
+         -t aimhubio/aimrocks:${{ matrix.manylinux-version }}
+
+      - name: Auditing wheels for ${{ matrix.manylinux-version }}
+        run: |
+         mkdir -p manylinux_dist/ && \
+         docker run --rm \
+         --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \
+         aimhubio/aimrocks:${{ matrix.manylinux-version }} \
+         bash -e /opt/aimrocks/docker/audit-wheels.sh
+         
+      - uses: actions/upload-artifact@v3
+        with:
+          path: manylinux_dist/*.whl
+
+  linux-dist-x86_64:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        manylinux-version: [ 'manylinux1_x86_64', 'manylinux2014_x86_64', 'manylinux_2_24_x86_64' ]
-    name: Linux wheels build (using Docker) ${{ matrix.manylinux-version }}
+        manylinux-version: [ 'manylinux1_x86_64', 'manylinux2010_x86_64', 'manylinux2014_x86_64', 'manylinux_2_24_x86_64' ]
+    name: ${{ matrix.manylinux-version }}
     steps:
       - name: Install Docker & images
         run: |
@@ -33,7 +106,9 @@ jobs:
         # Ignore the failure of a step and avoid terminating the job.
         continue-on-error: true
         with:
-          key: aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}
+          key: aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}-{hash}
+          restore-keys: |
+            aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}-
 
       - name: Building dependencies for ${{ matrix.manylinux-version }}
         run: |
@@ -62,23 +137,17 @@ jobs:
          aimhubio/aimrocks:${{ matrix.manylinux-version }} \
          bash -e /opt/aimrocks/docker/audit-wheels.sh
 
-      - name: Install dev dependencies
-        run: |
-          python -m pip install -r requirements.dev.txt
-
-      - name: Publish wheels
-        env:
-          PYPI_PASSWORD: ${{ secrets.pypi_password }}
-        run: |
-          python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" manylinux_dist/*
+      - uses: actions/upload-artifact@v3
+        with:
+          path: manylinux_dist/*.whl
 
   macos-deps:
     runs-on: m1
     strategy:
       fail-fast: true
       matrix:
         arch: ['arm64', 'x86_64']
-    name: Preparing dependencies for ${{ matrix.arch }} build
+    name: Preparing dependencies for ${{ matrix.arch }} Mac build
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.14
       AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }}
@@ -142,9 +211,7 @@ jobs:
       - name: Build and test wheels
         run: |
           arch -${{matrix.arch}} $PYTHON -m build
-
-      - name: Publish wheel
-        env:
-          PYPI_PASSWORD: ${{ secrets.pypi_password }}
-        run: |
-          $PYTHON -m twine upload --verbose -u __token__ -p "${PYPI_PASSWORD}" dist/*.whl
+          
+      - uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.whl
diff --git a/docker/audit-wheels.sh b/docker/audit-wheels.sh
@@ -18,7 +18,7 @@ cd /opt/aimrocks
 ls -lhatr dist
 
 $INTERNAL_PYTHON -m pip install --upgrade pip
-$INTERNAL_PYTHON -m pip install git+https://github.com/aimhubio/auditwheel.git@include-exclude
+$INTERNAL_PYTHON -m pip install git+https://github.com/aimhubio/auditwheel.git@include-exclude-new
 
 for whl in dist/*.whl
 do

diff --git a/setup.py b/setup.py
@@ -92,7 +92,7 @@
 
 setup(
     name="aimrocks",
-    version='0.3.1',
+    version='0.4.0',
     description='RocksDB wrapper implemented in Cython.',
     setup_requires=['setuptools>=25', 'Cython>=3.0.0a9'],
     packages=find_packages('./src'),