Post-release 2023.10.1 (#1243)

* Unpin dask/distributed for development * Update conda recipe * Linting * Bump gpuCI environments to 23.12 * Bump ucx-py to 0.35 * Re-enable linux-aarch64 builds * Make sure to publish Docker images on release * Amend DOCKER_PUSH conditional * Try some conda-forge tweaks to unblock linux-aarch64
dask-contrib · Oct 18, 2023 · 3972967 · 3972967
1 parent 88d9a8a
commit 3972967
Show file tree

Hide file tree

Showing 15 changed files with 119 additions and 33 deletions.
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -32,10 +32,38 @@ jobs:
       fail-fast: false
       matrix:
         python: ["3.8", "3.9", "3.10"]
-        # FIXME: aarch64 builds are consuming too much memory to run on GHA
-        # arch: ["linux-64", "linux-aarch64"]
-        arch: ["linux-64"]
+        arch: ["linux-64", "linux-aarch64"]
     steps:
+      - name: Manage disk space
+        if: matrix.arch == 'linux-aarch64'
+        run: |
+          sudo mkdir -p /opt/empty_dir || true
+          for d in \
+                    /opt/ghc \
+                    /opt/hostedtoolcache \
+                    /usr/lib/jvm \
+                    /usr/local/.ghcup \
+                    /usr/local/lib/android \
+                    /usr/local/share/powershell \
+                    /usr/share/dotnet \
+                    /usr/share/swift \
+                    ; do
+            sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
+          done
+          sudo apt-get purge -y -f firefox \
+                                    google-chrome-stable \
+                                    microsoft-edge-stable
+          sudo apt-get autoremove -y >& /dev/null
+          sudo apt-get autoclean -y >& /dev/null
+          sudo docker image prune --all --force
+          df -h
+      - name: Create swapfile
+        if: matrix.arch == 'linux-aarch64'
+        run: |
+          sudo fallocate -l 10GiB /swapfile || true
+          sudo chmod 600 /swapfile || true
+          sudo mkswap /swapfile || true
+          sudo swapon /swapfile || true
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -25,7 +25,7 @@ jobs:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     env:
-      DOCKER_PUSH: ${{ github.event_name == 'push' && github.repository == 'dask-contrib/dask-sql' }}
+      DOCKER_PUSH: ${{ contains(['push', 'release'], github.event_name) && github.repository == 'dask-contrib/dask-sql' }}
     strategy:
       fail-fast: false
       matrix:

diff --git a/.gitignore b/.gitignore
@@ -55,6 +55,7 @@ docs/source/_build/
 tests/unit/queries
 tests/unit/data
 target/*
+packages/*
 
 # Ignore development specific local testing files
 dev_tests

diff --git a/continuous_integration/environment-3.10-dev.yaml b/continuous_integration/environment-3.10-dev.yaml
@@ -4,7 +4,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0,<=2023.9.3
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1

diff --git a/continuous_integration/environment-3.9-dev.yaml b/continuous_integration/environment-3.9-dev.yaml
@@ -4,7 +4,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0,<=2023.9.3
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1

diff --git a/continuous_integration/gpuci/environment-3.10.yaml b/continuous_integration/gpuci/environment-3.10.yaml
@@ -8,7 +8,7 @@ channels:
 dependencies:
 - c-compiler
 - zlib
-- dask>=2022.3.0,<=2023.9.3
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
@@ -41,12 +41,12 @@ dependencies:
 - uvicorn>=0.13.4
 # GPU-specific requirements
 - cudatoolkit=11.5
-- cudf=23.10
-- cuml=23.10
-- dask-cudf=23.10
-- dask-cuda=23.10
+- cudf=23.12
+- cuml=23.12
+- dask-cudf=23.12
+- dask-cuda=23.12
 - ucx-proc=*=gpu
-- ucx-py=0.34
+- ucx-py=0.35
 - xgboost=*=rapidsai_py*
 - libxgboost=*=rapidsai_h*
 # TODO: unpin after RAPIDS 23.12 release

diff --git a/continuous_integration/gpuci/environment-3.9.yaml b/continuous_integration/gpuci/environment-3.9.yaml
@@ -8,7 +8,7 @@ channels:
 dependencies:
 - c-compiler
 - zlib
-- dask>=2022.3.0,<=2023.9.3
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
@@ -41,12 +41,12 @@ dependencies:
 - uvicorn>=0.13.4
 # GPU-specific requirements
 - cudatoolkit=11.5
-- cudf=23.10
-- cuml=23.10
-- dask-cudf=23.10
-- dask-cuda=23.10
+- cudf=23.12
+- cuml=23.12
+- dask-cudf=23.12
+- dask-cuda=23.12
 - ucx-proc=*=gpu
-- ucx-py=0.34
+- ucx-py=0.35
 - xgboost=*=rapidsai_py*
 - libxgboost=*=rapidsai_h*
 # TODO: unpin after RAPIDS 23.12 release

diff --git a/continuous_integration/recipe/build.sh b/continuous_integration/recipe/build.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -ex
+
+# See https://github.com/conda-forge/rust-feedstock/blob/master/recipe/build.sh for cc env explanation
+if [ "$c_compiler" = gcc ] ; then
+    case "$target_platform" in
+        linux-64) rust_env_arch=X86_64_UNKNOWN_LINUX_GNU ;;
+        linux-aarch64) rust_env_arch=AARCH64_UNKNOWN_LINUX_GNU ;;
+        linux-ppc64le) rust_env_arch=POWERPC64LE_UNKNOWN_LINUX_GNU ;;
+        *) echo "unknown target_platform $target_platform" ; exit 1 ;;
+    esac
+
+    export CARGO_TARGET_${rust_env_arch}_LINKER=$CC
+fi
+
+declare -a _xtra_maturin_args
+
+mkdir -p $SRC_DIR/.cargo
+
+if [ "$target_platform" = "osx-64" ] ; then
+    cat <<EOF >> $SRC_DIR/.cargo/config
+[target.x86_64-apple-darwin]
+linker = "$CC"
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+
+EOF
+
+    _xtra_maturin_args+=(--target=x86_64-apple-darwin)
+
+elif [ "$target_platform" = "osx-arm64" ] ; then
+    cat <<EOF >> $SRC_DIR/.cargo/config
+# Required for intermediate codegen stuff
+[target.x86_64-apple-darwin]
+linker = "$CC_FOR_BUILD"
+
+# Required for final binary artifacts for target
+[target.aarch64-apple-darwin]
+linker = "$CC"
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+
+EOF
+    _xtra_maturin_args+=(--target=aarch64-apple-darwin)
+
+    # This variable must be set to the directory containing the target's libpython DSO
+    export PYO3_CROSS_LIB_DIR=$PREFIX/lib
+
+    # xref: https://github.com/PyO3/pyo3/commit/7beb2720
+    export PYO3_PYTHON_VERSION=${PY_VER}
+
+    # xref: https://github.com/conda-forge/python-feedstock/issues/621
+    sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/os-patch.py
+    sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py
+fi
+
+maturin build -vv -j "${CPU_COUNT}" --release --strip --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}"
+
+"${PYTHON}" -m pip install $SRC_DIR/target/wheels/dask_sql*.whl --no-deps -vv
diff --git a/continuous_integration/recipe/conda_build_config.yaml b/continuous_integration/recipe/conda_build_config.yaml
@@ -1,8 +1,4 @@
-c_compiler_version:
-    - 11
 rust_compiler_version:
     - 1.69
-libprotobuf:
-    - 3
 maturin:
     - 1.1
diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml
@@ -13,32 +13,29 @@ source:
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
-  skip: true   # [py2k]
   entry_points:
     - dask-sql-server = dask_sql.server.app:main
     - dask-sql = dask_sql.cmd:main
   string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
-  script: RUST_BACKTRACE=full {{ PYTHON }} -m pip install . --no-deps -vv
 
 requirements:
   build:
     - python                                 # [build_platform != target_platform]
     - cross-python_{{ target_platform }}     # [build_platform != target_platform]
     - maturin                                # [build_platform != target_platform]
-    - libprotobuf                            # [build_platform != target_platform]
+    - libprotobuf
     - zlib                                   # [build_platform != target_platform]
     - {{ compiler('c') }}
     - {{ compiler('rust') }}
   host:
     - pip
     - python
     - maturin
-    - libprotobuf
     - zlib
     - xz  # [linux64]
   run:
     - python
-    - dask >=2022.3.0,<=2023.9.3
+    - dask >=2022.3.0
     - pandas >=1.4.0
     - fastapi >=0.92.0
     - httpx >=0.24.1

diff --git a/docker/conda.txt b/docker/conda.txt
@@ -1,5 +1,5 @@
 python>=3.8
-dask>=2022.3.0,<=2023.9.3
+dask>=2022.3.0
 pandas>=1.4.0
 jpype1>=1.0.2
 openjdk>=8

diff --git a/docker/main.dockerfile b/docker/main.dockerfile
@@ -16,7 +16,7 @@ RUN mamba install -y \
     # build requirements
     "maturin>=1.1,<1.2" \
     # core dependencies
-    "dask>=2022.3.0,<=2023.9.3" \
+    "dask>=2022.3.0" \
     "pandas>=1.4.0" \
     "fastapi>=0.92.0" \
     "httpx>=0.24.1" \

diff --git a/docs/environment.yml b/docs/environment.yml
@@ -7,7 +7,7 @@ dependencies:
   - sphinx>=4.0.0
   - sphinx-tabs
   - dask-sphinx-theme>=2.0.3
-  - dask>=2022.3.0,<=2023.9.3
+  - dask>=2022.3.0
   - pandas>=1.4.0
   - fugue>=0.7.3
   - fastapi>=0.92.0

diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
@@ -1,7 +1,7 @@
 sphinx>=4.0.0
 sphinx-tabs
 dask-sphinx-theme>=3.0.0
-dask>=2022.3.0,<=2023.9.3
+dask>=2022.3.0
 pandas>=1.4.0
 fugue>=0.7.3
 fastapi>=0.92.0

diff --git a/pyproject.toml b/pyproject.toml
@@ -26,8 +26,8 @@ classifiers = [
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "dask[dataframe]>=2022.3.0,<=2023.9.3",
-    "distributed>=2022.3.0,<=2023.9.3",
+    "dask[dataframe]>=2022.3.0",
+    "distributed>=2022.3.0",
     "pandas>=1.4.0",
     "fastapi>=0.92.0",
     "httpx>=0.24.1",