github.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

{% import 'macros.jinja' as macros with context %}

{{ macros.github_header() }}

jobs:

  build-cpp-ubuntu:
    {% set arch = '${{ matrix.platform.arch }}' %}
    name: Build C++ libraries Ubuntu {{ arch }}
    runs-on: {{ '${{ matrix.platform.runs_on }}' }}
    env:
      # architecture name used for archery build
      ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
      ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
      ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - runs_on: ["ubuntu-latest"]
            arch: "x86_64"
            archery_arch: "amd64"
            archery_arch_alias: "x86_64"
            archery_arch_short: "amd64"
          - runs_on: ["self-hosted", "Linux", "arm64"]
            arch: "aarch_64"
            archery_arch: "arm64v8"
            archery_arch_alias: "aarch64"
            archery_arch_short: "arm64"
    steps:
      {{ macros.github_checkout_arrow()|indent }}
      {{ macros.github_free_space()|indent }}
      {{ macros.github_install_archery()|indent }}
      - name: Build C++ libraries
        env:
        {{ macros.github_set_sccache_envvars()|indent(8) }}
        run: |
          archery docker run \
            -e ARROW_JAVA_BUILD=OFF \
            -e ARROW_JAVA_TEST=OFF \
            java-jni-manylinux-2014
      - name: Compress into single artifact to keep directory structure
        run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ubuntu-shared-lib-{{ arch }}
          path: arrow-shared-libs-linux-{{ arch }}.tar.gz
    {% if arrow.is_default_branch() %}
      {{ macros.github_login_dockerhub()|indent }}
      - name: Push Docker image
        shell: bash
        run: archery docker push java-jni-manylinux-2014
    {% endif %}

  build-cpp-macos:
    {% set arch = '${{ matrix.platform.arch }}' %}
    name: Build C++ libraries macOS {{ arch }}
    runs-on: {{ '${{ matrix.platform.runs_on }}' }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - { runs_on: ["macos-13"], arch: "x86_64"}
          - { runs_on: ["macos-14"], arch: "aarch_64" }
    env:
      MACOSX_DEPLOYMENT_TARGET: "14.0"
    steps:
      {{ macros.github_checkout_arrow()|indent }}
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.12
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]
      - name: Install dependencies
        run: |
          # We want to use llvm@14 to avoid shared z3
          # dependency. llvm@14 doesn't depend on z3 and llvm depends
          # on z3. And Homebrew's z3 provides only shared library. It
          # doesn't provides static z3 because z3's CMake doesn't accept
          # building both shared and static libraries at once.
          # See also: Z3_BUILD_LIBZ3_SHARED in
          # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
          #
          # If llvm is installed, Apache Arrow C++ uses llvm rather than
          # llvm@14 because llvm is newer than llvm@14.
          brew uninstall llvm || :

          # Ensure updating python@XXX with the "--overwrite" option.
          # If python@XXX is updated without "--overwrite", it causes
          # a conflict error. Because Python 3 installed not by
          # Homebrew exists in /usr/local on GitHub Actions. If
          # Homebrew's python@XXX is updated without "--overwrite", it
          # tries to replace /usr/local/bin/2to3 and so on and causes
          # a conflict error.
          brew update
          for python_package in $(brew list | grep python@); do
            brew install --overwrite ${python_package}
          done
          brew install --overwrite python

          brew bundle --file=arrow/cpp/Brewfile
          # We want to link aws-sdk-cpp statically but Homebrew's
          # aws-sdk-cpp provides only shared library. If we have
          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
          brew uninstall aws-sdk-cpp
          # We want to use bundled RE2 for static linking. If
          # Homebrew's RE2 is installed, its header file may be used.
          # We uninstall Homebrew's RE2 to ensure using bundled RE2.
          brew uninstall grpc || : # gRPC depends on RE2
          brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
          brew uninstall re2
          # We want to use bundled Protobuf for static linking. If
          # Homebrew's Protobuf is installed, its library file may be
          # used on test  We uninstall Homebrew's Protobuf to ensure using
          # bundled Protobuf.
          brew uninstall protobuf

          brew bundle --file=arrow/java/Brewfile
      - name: Build C++ libraries
        env:
        {{ macros.github_set_sccache_envvars()|indent(8) }}
        run: |
          set -e
          # make brew Java available to CMake
          export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
          arrow/ci/scripts/java_jni_macos_build.sh \
            $GITHUB_WORKSPACE/arrow \
            $GITHUB_WORKSPACE/arrow/cpp-build \
            $GITHUB_WORKSPACE/arrow/java-dist
      - name: Compress into single artifact to keep directory structure
        run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: macos-shared-lib-{{ arch }}
          path: arrow-shared-libs-macos-{{ arch }}.tar.gz

  build-cpp-windows:
    name: Build C++ libraries Windows
    runs-on: windows-2019
    steps:
      {{ macros.github_checkout_arrow()|indent }}
      - name: Set up Java
        uses: actions/setup-java@v3
        with:
          java-version: '11'
          distribution: 'temurin'
      - name: Download Timezone Database
        shell: bash
        run: arrow/ci/scripts/download_tz_database.sh
      - name: Install sccache
        shell: bash
        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
      - name: Build C++ libraries
        shell: cmd
        env:
        {{ macros.github_set_sccache_envvars()|indent(8) }}
        run: |
          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          REM For ORC
          set TZDIR=/c/msys64/usr/share/zoneinfo
          bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
      - name: Compress into single artifact to keep directory structure
        shell: bash
        run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: windows-shared-lib
          path: arrow-shared-libs-windows.tar.gz

  package-jars:
    name: Build jar files
    runs-on: macos-latest
    needs:
      - build-cpp-ubuntu
      - build-cpp-macos
      - build-cpp-windows
    steps:
      {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
      - name: Download Libraries
        uses: actions/download-artifact@v4
        with:
          path: artifacts
      - name: Decompress artifacts
        run: |
          mv artifacts/*/*.tar.gz .
          tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
          tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
          tar -xvzf arrow-shared-libs-windows.tar.gz
      - name: Test that shared libraries exist
        run: |
          set -x

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
          test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
          test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so

          test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
          test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
          test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
          test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
          test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
          test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib

          test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
          test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
          test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
          test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
          test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
      - name: Build bundled jar
        env:
          MAVEN_ARGS: >-
            --no-transfer-progress
        run: |
          set -e
          pushd arrow/java
          mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }}
          mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom
          popd
          arrow/ci/scripts/java_full_build.sh \
            $GITHUB_WORKSPACE/arrow \
            $GITHUB_WORKSPACE/arrow/java-dist
      {{ macros.github_upload_releases(["arrow/java-dist/*.jar",
                                        "arrow/java-dist/*.json",
                                        "arrow/java-dist/*.pom",
                                        "arrow/java-dist/*.xml",
                                        "arrow/java-dist/*.zip"])|indent }}