Merge branch 'main' into dependabot/github_actions/g0-9ffd8ce23e

coiled · Aug 5, 2024 · 3b1239f · 3b1239f
2 parents 00ac1c0 + a8a31b1
commit 3b1239f
Show file tree

Hide file tree

Showing 45 changed files with 1,161 additions and 544 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.github/workflows/ab_tests.yml b/.github/workflows/ab_tests.yml
@@ -69,7 +69,6 @@ jobs:
           cp AB_baseline.conda.yaml AB_null_hypothesis.conda.yaml
           cp AB_baseline.dask.yaml AB_null_hypothesis.dask.yaml
           cp AB_baseline.cluster.yaml AB_null_hypothesis.cluster.yaml
-          cp AB_baseline.requirements.in AB_null_hypothesis.requirements.in
 
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v3
@@ -79,11 +78,8 @@ jobs:
           condarc-file: ci/condarc
           environment-file: AB_environments/${{ matrix.runtime-version }}.conda.yaml
 
-      - name: Pip Compile
-        run: pip-compile AB_environments/${{ matrix.runtime-version }}.requirements.in
-
-      - name: Add Environment dependencies
-        run: python -m pip install -r AB_environments/${{ matrix.runtime-version }}.requirements.txt
+      - name: Add test dependencies
+        run: mamba env update --file ci/environment-test.yml
 
       - name: Dump environment
         run: |

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -32,44 +32,40 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python_version: ["3.9"]
+        python_version: ["3.10"]
         pytest_args: [tests --ignore=tests/tpch]
-        requirements: [ci/requirements-2nightly.in]
+        extra-env: [""]
         name_prefix: [tests]
         include:
           # Run stability tests on the lowest and highest versions of Python only
           # These are temporarily redundant with the current global python_version
           # - pytest_args: tests/stability
-          #   python_version: "3.9"
+          #   python_version: "3.10"
           #   os: ubuntu-latest
           #   name_prefix: stability
           - pytest_args: tests/stability
             python_version: "3.11"
             os: ubuntu-latest
-            requirements: ci/requirements-2nightly.in
             name_prefix: stability
           # Run stability tests on Python Windows and MacOS (latest py39 only)
           - pytest_args: tests/stability
-            python_version: "3.9"
+            python_version: "3.10"
             os: windows-latest
             name_prefix: stability
-            requirements: ci/requirements-2nightly.in
           - pytest_args: tests/stability
-            python_version: "3.9"
+            python_version: "3.10"
             os: macos-latest
             name_prefix: stability
-            requirements: ci/requirements-2nightly.in
-          # Special environments
           - pytest_args: tests/workflows/test_snowflake.py
-            python_version: "3.9"
+            python_version: "3.10"
             os: ubuntu-latest
             name_prefix: snowflake
-            requirements: ci/requirements-2snowflake.in
+            extra-env: ci/environment-snowflake.yml
           - pytest_args: tests/tpch
-            python_version: "3.9"
+            python_version: "3.10"
             os: ubuntu-latest
+            extra-env: ci/environment-tpch-nondask.yml
             name_prefix: tpch
-            requirements: ci/requirements-2tpch-non-dask.in
 
     steps:
       - name: Checkout
@@ -84,13 +80,15 @@ jobs:
           python-version: ${{ matrix.python_version }}
           environment-file: ci/environment.yml
 
-      - name: Pip Compile
-        run: pip-compile ${{ matrix.requirements }}
+      - name: Add extra packages to environment
+        if: ${{ matrix.extra-env != '' }}
+        run: mamba env update --file ${{ matrix.extra-env }}
 
-      - name: Add Environment dependencies
-        run: |
-          IN_FILE=${{ matrix.requirements }}
-          python -m pip install -r ${IN_FILE%.*}.txt
+      - name: Upgrade dask to git tip
+        run: mamba env update --file ci/environment-git-tip.yml
+
+      - name: Add test dependencies
+        run: mamba env update --file ci/environment-test.yml
 
       - name: Reconfigure pytest-timeout
         shell: bash -l {0}
@@ -135,6 +133,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
+          PYTHON_STUB_PAT: ${{ secrets.PYTHON_STUB_PAT }}
           SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
           SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
           SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}

diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml
@@ -57,11 +57,14 @@ jobs:
         python-version: "3.9"
         environment-file: ci/environment.yml
 
-    - name: Pip Compile
-      run: pip-compile ci/requirements-2tpch-non-dask.in
+    - name: Add TPC-H non-dask dependencies
+      run: mamba env update --file ci/environment-tpch-nondask.yml
 
-    - name: Add Environment dependencies
-      run: python -m pip install -r ci/requirements-2tpch-non-dask.txt
+    - name: Upgrade dask to git tip
+      run: mamba env update --file ci/environment-git-tip.yml
+
+    - name: Add test dependencies
+      run: mamba env update --file ci/environment-test.yml
 
     - name: Dump environment
       run: |

diff --git a/.gitignore b/.gitignore
@@ -144,7 +144,3 @@ tpch-data/
 
 # .visualize() output of dask collections
 mydask.html
-
-# pip-compiled files
-ci/requirements-*.txt
-AB_environments/AB_*.requirements.txt
diff --git a/AB_environments/AB_baseline.cluster.yaml b/AB_environments/AB_baseline.cluster.yaml
@@ -1,14 +1,6 @@
-# Environment file for A/B testing, used to define cluster creation options for
-# an environment.
-#
-# You may change contents and duplicate to other environments, but do not rename
-# the AB_baseline files.
-# Please read README.md for more information.
+# Special environment file for A/B testing, used to define cluster creation options for
+# the baseline environment.
+# Change contents, but do not rename.
 
 # Overrides ../cluster_kwargs.yaml.
 # Leave empty if you don't want to override anything.
-
-# EXAMPLE - uncomment/change as needed
-# small_cluster:
-#   n_workers: 5
-#   worker_vm_types: [m6i.xlarge]  # 4CPU, 16GiB
diff --git a/AB_environments/AB_baseline.conda.yaml b/AB_environments/AB_baseline.conda.yaml
@@ -1,16 +1,56 @@
-# Environment file for A/B testing, used to define the conda environment.
-# This is just for non-pip packages - see AB_baseline.requirements.in for
-# everything else.
-#
-# You may change contents and duplicate to other environments, but do not rename
-# the AB_baseline files.
-# Please read README.md for more information.
+# Special environment file for A/B testing, used to define the conda environment for the
+# baseline environment.
+# Change contents, but do not rename.
 
 channels:
   - conda-forge
 dependencies:
-  - python =3.9  # Single '=' means latest patch version available
-  - openssl>1.1.0g
+  - python =3.10  # Single '=' means latest patch version available
+  # Copy-paste from ci/environment.yml
   - pip
-  - pip-tools
-  # NOTE: Packages are defined in AB_baseline.requirements.in
+  - coiled >=0.2.54
+  - numpy ==1.26.4
+  - pandas ==2.2.2
+  - dask ==2024-07.1
+  - distributed ==2024-07.1
+  - dask-expr ==1.1.9
+  - dask-labextension ==7.0.0
+  - dask-ml ==2024.4.4
+  - fsspec ==2024.6.1
+  - s3fs ==2024.6.1
+  - gcsfs ==2024.6.1
+  - pyarrow ==17.0.0
+  - jupyterlab ==4.2.4
+  - lz4 ==4.3.3
+  - ipywidgets ==8.1.3
+  - numba ==0.60.0
+  - scikit-learn ==1.5.1
+  - ipycytoscape ==1.3.3
+  - click ==8.1.7
+  - xarray ==2024.07.0
+  - zarr ==2.18.2
+  - cftime ==1.6.4
+  - msgpack-python
+  - cloudpickle ==3.0.0
+  - tornado ==6.4.1
+  - toolz ==0.12.1
+  - zict ==3.0.0
+  - xgboost ==2.1.1
+  - optuna ==3.6.1
+  - optuna-integration ==3.6.0
+  - scipy ==1.14.0
+  - sqlalchemy ==2.0.31
+  - pynvml ==11.5.3
+  - bokeh ==3.5.1
+  - gilknocker ==0.4.1
+  - openssl >1.1.0g
+  # End copy-paste
+
+  - pip:
+    # Make sure you install dask and distributed either both from pip or both from
+    # conda. You may alternatively point to your own git fork (but make sure you
+    # sync'ed tags!)
+    # Read README.md for troubleshooting.
+    # - git+https://github.com/dask/dask@191d39177009d2cce25b818878118e35329b6db3
+    # - git+https://github.com/dask/distributed@0304fb6e665e36abf9e3086173cccd36e29ae84d
+    # - git+https://github.com/dask-contrib/dask-expr@9f765764da3f518ddd4c896c98b8a40a979a5553
diff --git a/AB_environments/AB_baseline.dask.yaml b/AB_environments/AB_baseline.dask.yaml
@@ -1,12 +1,7 @@
-# Environment file for A/B testing, used to define dask config options
-# (overriding the built-in config).
+# Special environment file for A/B testing, used to define dask config options
+# (overriding the built-in config) for the baseline environment.
+# Change contents, but do not rename.
 # Leave empty if you don't want to override anything.
-#
-# You may change contents and duplicate to other environments, but do not rename
-# the AB_baseline files.
-# Please read README.md for more information.
-
-# EXAMPLE - uncomment/change as needed
-# distributed:
-#   scheduler:
-#     worker-saturation: 1.2
+dask:
+  dataframe:
+    query-planning: True
diff --git a/AB_environments/AB_baseline.requirements.in b/AB_environments/AB_baseline.requirements.in
diff --git a/AB_environments/AB_sample.cluster.yaml b/AB_environments/AB_sample.cluster.yaml
@@ -0,0 +1,14 @@
+# Sample cluster creation options file for A/B testing.
+# Change contents/delete/rename as needed.
+
+# Every A/B environment *must* present these three files:
+# - AB_<name>.conda.yaml
+# - AB_<name>.dask.yaml
+# - AB_<name>.cluster.yaml
+
+# Overrides ../cluster_kwargs.yaml.
+# Leave empty if you don't want to override anything.
+
+# small_cluster:
+#   n_workers: 5
+#   worker_vm_types: [m6i.xlarge]  # 4CPU, 16GiB
diff --git a/AB_environments/AB_sample.conda.yaml b/AB_environments/AB_sample.conda.yaml
@@ -0,0 +1,62 @@
+# Sample conda environment file for A/B testing.
+# Change contents/delete/rename as needed.
+
+# Every A/B environment *must* present these three files:
+# - AB_<name>.conda.yaml
+# - AB_<name>.dask.yaml
+# - AB_<name>.cluster.yaml
+
+# You should always start from a copy-paste from AB_baseline.conda.yaml
+
+channels:
+  - conda-forge
+dependencies:
+  - python =3.10  # Single '=' means latest patch version available
+  # Copy-paste from ci/environment.yml
+  - pip
+  - coiled >=0.2.54
+  - numpy ==1.26.4
+  - pandas ==2.2.2
+  - dask ==2024-07.1
+  - distributed ==2024-07.1
+  - dask-expr ==1.1.9
+  - dask-labextension ==7.0.0
+  - dask-ml ==2024.4.4
+  - fsspec ==2024.6.1
+  - s3fs ==2024.6.1
+  - gcsfs ==2024.6.1
+  - pyarrow ==17.0.0
+  - jupyterlab ==4.2.4
+  - lz4 ==4.3.3
+  - ipywidgets ==8.1.3
+  - numba ==0.60.0
+  - scikit-learn ==1.5.1
+  - ipycytoscape ==1.3.3
+  - click ==8.1.7
+  - xarray ==2024.07.0
+  - zarr ==2.18.2
+  - cftime ==1.6.4
+  - msgpack-python
+  - cloudpickle ==3.0.0
+  - tornado ==6.4.1
+  - toolz ==0.12.1
+  - zict ==3.0.0
+  - xgboost ==2.1.1
+  - optuna ==3.6.1
+  - optuna-integration ==3.6.0
+  - scipy ==1.14.0
+  - sqlalchemy ==2.0.31
+  - pynvml ==11.5.3
+  - bokeh ==3.5.1
+  - gilknocker ==0.4.1
+  - openssl >1.1.0g
+  # End copy-paste
+
+  - pip:
+    # Make sure you install dask and distributed either both from pip or both from
+    # conda. You may alternatively point to your own git fork (but make sure you
+    # sync'ed tags!)
+    # Read README.md for troubleshooting.
+    - git+https://github.com/dask/dask@191d39177009d2cce25b818878118e35329b6db3
+    - git+https://github.com/dask/distributed@0304fb6e665e36abf9e3086173cccd36e29ae84d
+    - git+https://github.com/dask-contrib/dask-expr@9f765764da3f518ddd4c896c98b8a40a979a5553
diff --git a/AB_environments/AB_sample.dask.yaml b/AB_environments/AB_sample.dask.yaml
@@ -0,0 +1,16 @@
+# Sample dask config file for A/B testing.
+# Change contents/delete/rename as needed.
+
+# Every A/B environment *must* present these three files:
+# - AB_<name>.conda.yaml
+# - AB_<name>.dask.yaml
+# - AB_<name>.cluster.yaml
+
+# Leave empty if you don't want to override anything.
+
+# distributed:
+#   scheduler:
+#     worker-saturation: 1.2
+dask:
+  dataframe:
+    query-planning: True