diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index da783dcd2..c53a973de 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -32,10 +32,38 @@ jobs: fail-fast: false matrix: python: ["3.8", "3.9", "3.10"] - # FIXME: aarch64 builds are consuming too much memory to run on GHA - # arch: ["linux-64", "linux-aarch64"] - arch: ["linux-64"] + arch: ["linux-64", "linux-aarch64"] steps: + - name: Manage disk space + if: matrix.arch == 'linux-aarch64' + run: | + sudo mkdir -p /opt/empty_dir || true + for d in \ + /opt/ghc \ + /opt/hostedtoolcache \ + /usr/lib/jvm \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + ; do + sudo rsync --stats -a --delete /opt/empty_dir/ $d || true + done + sudo apt-get purge -y -f firefox \ + google-chrome-stable \ + microsoft-edge-stable + sudo apt-get autoremove -y >& /dev/null + sudo apt-get autoclean -y >& /dev/null + sudo docker image prune --all --force + df -h + - name: Create swapfile + if: matrix.arch == 'linux-aarch64' + run: | + sudo fallocate -l 10GiB /swapfile || true + sudo chmod 600 /swapfile || true + sudo mkswap /swapfile || true + sudo swapon /swapfile || true - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ba343dbd5..a0c95beac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - DOCKER_PUSH: ${{ github.event_name == 'push' && github.repository == 'dask-contrib/dask-sql' }} + DOCKER_PUSH: ${{ contains(['push', 'release'], github.event_name) && github.repository == 'dask-contrib/dask-sql' }} strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index d41df8a68..18b2a3d83 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ docs/source/_build/ tests/unit/queries tests/unit/data target/* +packages/* # Ignore development specific local testing files dev_tests diff --git a/continuous_integration/environment-3.10-dev.yaml b/continuous_integration/environment-3.10-dev.yaml index 100e3d0a0..4589cc1b5 100644 --- a/continuous_integration/environment-3.10-dev.yaml +++ b/continuous_integration/environment-3.10-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - c-compiler -- dask>=2022.3.0,<=2023.9.3 +- dask>=2022.3.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 diff --git a/continuous_integration/environment-3.9-dev.yaml b/continuous_integration/environment-3.9-dev.yaml index 410c504b1..f807d2e87 100644 --- a/continuous_integration/environment-3.9-dev.yaml +++ b/continuous_integration/environment-3.9-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - c-compiler -- dask>=2022.3.0,<=2023.9.3 +- dask>=2022.3.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 diff --git a/continuous_integration/gpuci/environment-3.10.yaml b/continuous_integration/gpuci/environment-3.10.yaml index 8c0312788..c8ab54c9d 100644 --- a/continuous_integration/gpuci/environment-3.10.yaml +++ b/continuous_integration/gpuci/environment-3.10.yaml @@ -8,7 +8,7 @@ channels: dependencies: - c-compiler - zlib -- dask>=2022.3.0,<=2023.9.3 +- dask>=2022.3.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 @@ -41,12 +41,12 @@ dependencies: - uvicorn>=0.13.4 # GPU-specific requirements - cudatoolkit=11.5 -- cudf=23.10 -- cuml=23.10 -- dask-cudf=23.10 -- dask-cuda=23.10 +- cudf=23.12 +- cuml=23.12 +- dask-cudf=23.12 +- dask-cuda=23.12 - ucx-proc=*=gpu -- ucx-py=0.34 +- ucx-py=0.35 - xgboost=*=rapidsai_py* - libxgboost=*=rapidsai_h* # TODO: unpin after RAPIDS 23.12 release diff --git a/continuous_integration/gpuci/environment-3.9.yaml b/continuous_integration/gpuci/environment-3.9.yaml index 57ee545f5..38b380f92 100644 --- a/continuous_integration/gpuci/environment-3.9.yaml +++ b/continuous_integration/gpuci/environment-3.9.yaml @@ -8,7 +8,7 @@ channels: dependencies: - c-compiler - zlib -- dask>=2022.3.0,<=2023.9.3 +- dask>=2022.3.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 @@ -41,12 +41,12 @@ dependencies: - uvicorn>=0.13.4 # GPU-specific requirements - cudatoolkit=11.5 -- cudf=23.10 -- cuml=23.10 -- dask-cudf=23.10 -- dask-cuda=23.10 +- cudf=23.12 +- cuml=23.12 +- dask-cudf=23.12 +- dask-cuda=23.12 - ucx-proc=*=gpu -- ucx-py=0.34 +- ucx-py=0.35 - xgboost=*=rapidsai_py* - libxgboost=*=rapidsai_h* # TODO: unpin after RAPIDS 23.12 release diff --git a/continuous_integration/recipe/build.sh b/continuous_integration/recipe/build.sh new file mode 100644 index 000000000..3750311d7 --- /dev/null +++ b/continuous_integration/recipe/build.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -ex + +# See https://github.com/conda-forge/rust-feedstock/blob/master/recipe/build.sh for cc env explanation +if [ "$c_compiler" = gcc ] ; then + case "$target_platform" in + linux-64) rust_env_arch=X86_64_UNKNOWN_LINUX_GNU ;; + linux-aarch64) rust_env_arch=AARCH64_UNKNOWN_LINUX_GNU ;; + linux-ppc64le) rust_env_arch=POWERPC64LE_UNKNOWN_LINUX_GNU ;; + *) echo "unknown target_platform $target_platform" ; exit 1 ;; + esac + + export CARGO_TARGET_${rust_env_arch}_LINKER=$CC +fi + +declare -a _xtra_maturin_args + +mkdir -p $SRC_DIR/.cargo + +if [ "$target_platform" = "osx-64" ] ; then + cat <> $SRC_DIR/.cargo/config +[target.x86_64-apple-darwin] +linker = "$CC" +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +EOF + + _xtra_maturin_args+=(--target=x86_64-apple-darwin) + +elif [ "$target_platform" = "osx-arm64" ] ; then + cat <> $SRC_DIR/.cargo/config +# Required for intermediate codegen stuff +[target.x86_64-apple-darwin] +linker = "$CC_FOR_BUILD" + +# Required for final binary artifacts for target +[target.aarch64-apple-darwin] +linker = "$CC" +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +EOF + _xtra_maturin_args+=(--target=aarch64-apple-darwin) + + # This variable must be set to the directory containing the target's libpython DSO + export PYO3_CROSS_LIB_DIR=$PREFIX/lib + + # xref: https://github.com/PyO3/pyo3/commit/7beb2720 + export PYO3_PYTHON_VERSION=${PY_VER} + + # xref: https://github.com/conda-forge/python-feedstock/issues/621 + sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/os-patch.py + sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py +fi + +maturin build -vv -j "${CPU_COUNT}" --release --strip --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}" + +"${PYTHON}" -m pip install $SRC_DIR/target/wheels/dask_sql*.whl --no-deps -vv diff --git a/continuous_integration/recipe/conda_build_config.yaml b/continuous_integration/recipe/conda_build_config.yaml index df3cde4e1..aa6e0d578 100644 --- a/continuous_integration/recipe/conda_build_config.yaml +++ b/continuous_integration/recipe/conda_build_config.yaml @@ -1,8 +1,4 @@ -c_compiler_version: - - 11 rust_compiler_version: - 1.69 -libprotobuf: - - 3 maturin: - 1.1 diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml index c3c2cff6c..fbe498eda 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipe/meta.yaml @@ -13,19 +13,17 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} - skip: true # [py2k] entry_points: - dask-sql-server = dask_sql.server.app:main - dask-sql = dask_sql.cmd:main string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script: RUST_BACKTRACE=full {{ PYTHON }} -m pip install . --no-deps -vv requirements: build: - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - maturin # [build_platform != target_platform] - - libprotobuf # [build_platform != target_platform] + - libprotobuf - zlib # [build_platform != target_platform] - {{ compiler('c') }} - {{ compiler('rust') }} @@ -33,12 +31,11 @@ requirements: - pip - python - maturin - - libprotobuf - zlib - xz # [linux64] run: - python - - dask >=2022.3.0,<=2023.9.3 + - dask >=2022.3.0 - pandas >=1.4.0 - fastapi >=0.92.0 - httpx >=0.24.1 diff --git a/docker/conda.txt b/docker/conda.txt index bc2dd29d1..5083ebd85 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,5 +1,5 @@ python>=3.8 -dask>=2022.3.0,<=2023.9.3 +dask>=2022.3.0 pandas>=1.4.0 jpype1>=1.0.2 openjdk>=8 diff --git a/docker/main.dockerfile b/docker/main.dockerfile index 2283a8791..f17e0181f 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -16,7 +16,7 @@ RUN mamba install -y \ # build requirements "maturin>=1.1,<1.2" \ # core dependencies - "dask>=2022.3.0,<=2023.9.3" \ + "dask>=2022.3.0" \ "pandas>=1.4.0" \ "fastapi>=0.92.0" \ "httpx>=0.24.1" \ diff --git a/docs/environment.yml b/docs/environment.yml index 4a52187c5..ca9b23a34 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -7,7 +7,7 @@ dependencies: - sphinx>=4.0.0 - sphinx-tabs - dask-sphinx-theme>=2.0.3 - - dask>=2022.3.0,<=2023.9.3 + - dask>=2022.3.0 - pandas>=1.4.0 - fugue>=0.7.3 - fastapi>=0.92.0 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 741d17a3e..cce9cb599 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,7 +1,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=3.0.0 -dask>=2022.3.0,<=2023.9.3 +dask>=2022.3.0 pandas>=1.4.0 fugue>=0.7.3 fastapi>=0.92.0 diff --git a/pyproject.toml b/pyproject.toml index b70157800..3caa92ddb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ readme = "README.md" requires-python = ">=3.8" dependencies = [ - "dask[dataframe]>=2022.3.0,<=2023.9.3", - "distributed>=2022.3.0,<=2023.9.3", + "dask[dataframe]>=2022.3.0", + "distributed>=2022.3.0", "pandas>=1.4.0", "fastapi>=0.92.0", "httpx>=0.24.1",