diff --git a/.github/workflows/arm64.yml b/.github/workflows/arm64.yml new file mode 100644 index 0000000..ec5495b --- /dev/null +++ b/.github/workflows/arm64.yml @@ -0,0 +1,220 @@ +name: ARM64_CI + +on: + workflow_dispatch: + inputs: + upload_wheel: + description: 'whether/where to upload the wheel' + required: false + default: 'no' + type: 'choice' + options: ['no', 'pypi', 'testpypi'] + + publish_npm: + description: 'whether to publish the npm packages' + required: false + default: 'no' + type: 'choice' + options: ['no', 'yes'] + + push: + branches: + - arm64 + pull_request: + branches: + - main + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + # - name: Ubuntu x86_64 + # os: ubuntu-latest + # cibw_archs: x86_64 + - name: ARM64 - Emulated + os: ubuntu-latest + cibw_archs: "aarch64" + # - name: Windows + # os: windows-latest + # cibw_archs: "auto" + # - name: Macos 13 + # os: macos-1 + # cibw_archs: "auto" + # - name: Macos 14 + # os: macos-14 + # cibw_archs: "auto" + # os: [ubuntu-latest, macos-13, macos-14] + # os: [windows-latest] + + steps: + - name: Set up QEMU + if: matrix.cibw_archs == 'aarch64' + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + + - uses: actions/checkout@v4 + + - uses: benjlevesque/short-sha@v3.0 + id: short_sha + + # Used to host cibuildwheel + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.19.2 pytest apsw numpy + + - uses: ilammy/msvc-dev-cmd@v1 + + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v2 + with: + cmake-version: '3.29.6' + + - name: Setup Ninja + # For now this action doesn't support arm64 + if: matrix.cibw_archs != 'aarch64' + uses: seanmiddleditch/gha-setup-ninja@v5 + with: + version: 1.11.1 + + - name: Bootstrap vcpkg + run: | + git submodule update --init --recursive + python bootstrap_vcpkg.py + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.cibw_archs }} + + - uses: actions/upload-artifact@v4 + with: + name: vectorlite-wheel-${{ matrix.os }}-${{ steps.short_sha.outputs.sha }} + path: ./wheelhouse/*.whl + + - name: Get release version + if: startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/v') + shell: bash + run: | + echo "RELEASE_VERSION=$(echo ${{ github.ref_name }} | sed -E 's/^release\///')" >> $GITHUB_ENV + + - name: Upload wheels to release assets + if: ${{ github.event_name == 'workflow_dispatch' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: + gh release upload ${{ env.RELEASE_VERSION }} ./wheelhouse/*.whl --clobber + + - name: Run python examples + shell: bash + run: | + + for wheel in wheelhouse/*.whl; do + echo "Running examples for $wheel" + python -m pip install $wheel --force-reinstall + python -m pip install -r examples/requirements.txt + for example in examples/*.py; do + echo "Running $example" + python $example + done + done + + - name: Run benchmark + working-directory: benchmark + shell: bash + run: | + python -m pip install -r requirements.txt + python benchmark.py + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ matrix.os }}-${{ steps.short_sha.outputs.sha }} + path: ./benchmark/vector*.png + + + upload_wheels: + name: Upload wheels + if: ${{ github.event.inputs.upload_wheel != 'no' && github.event_name != 'pull_request' }} + needs: build_wheels + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: benjlevesque/short-sha@v3.0 + id: short_sha + + # Download all artifacts + - uses: actions/download-artifact@v4 + with: + path: ./wheelhouse + + - name: Upload to test.pypi.org + if: ${{ github.event.inputs.upload_wheel == 'testpypi' }} + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: pipx run twine upload --repository testpypi wheelhouse/**/*.whl + + - name: Upload to pypi.org + if: ${{ github.event.inputs.upload_wheel == 'pypi' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: pipx run twine upload wheelhouse/**/*.whl + + - name: Fail if uploading to pypi.org without a tag + if: ${{ github.event.inputs.upload_wheel == 'pypi' && !(startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} + run: | + echo "Error: Uploading to pypi.org requires a tag" + exit 1 + + publish_npm_pkgs: + name: Upload npm packages + if: ${{ github.event.inputs.publish_npm == 'yes' && github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} + needs: build_wheels + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: benjlevesque/short-sha@v3.0 + id: short_sha + + # Download all artifacts + - uses: actions/download-artifact@v4 + with: + path: ./wheelhouse + + # extract vectorlite from wheels and copy to nodejs bindings directory + - name: unzip wheels + run: | + sh extract_wheels.sh + + - uses: actions/setup-node@v4 + with: + node-version: lts/* + registry-url: 'https://registry.npmjs.org' + + - name: Test locally + working-directory: bindings/nodejs/packages/vectorlite + run: | + npm i -D + npm link ../vectorlite-linux-x64 + npm run test + + - name: Publish to npm + working-directory: bindings/nodejs + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + run: | + mv package.json.tpl package.json + npm publish --workspaces --access public \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b929d75..456285c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,4 +14,15 @@ environment = {MACOSX_DEPLOYMENT_TARGET = "10.15"} # 10.15 is the minimum versio # todo: support musllinux [tool.cibuildwheel.linux] -before-build = "yum install -y ninja-build" \ No newline at end of file +before-build = "yum install -y ninja-build" + +[[tool.cibuildwheel.overrides]] +select = "*aarch64*" +environment = {VCPKG_FORCE_SYSTEM_BINARIES="1"} +before-build = [ + "yum update", + "yum install -y curl zip unzip tar", + "git clone https://github.com/ninja-build/ninja.git", + "cd ninja && python3 configure.py --bootstrap", + "mv ninja /usr/bin/ && cd .. && rm -rf ninja" + ] \ No newline at end of file