Merge branch 'main' of github.com:dask/distributed into stealing_scaling

.coveragerc

@@ -1,19 +1,30 @@
 [run]
-include = 
-    distributed/*
+source =
+    distributed
 omit =
     distributed/tests/test*
-    distributed/hdfs.py
-    distributed/cluster.py
     distributed/*/tests/test*
-    distributed/compatibility.py
     distributed/cli/utils.py
-    distributed/utils_test.py
+    distributed/cli/dask_spec.py
     distributed/deploy/ssh.py
-    distributed/_ipython_utils.py
+    distributed/_version.py
+    distributed/pytest_resourceleaks.py
+    distributed/comm/ucx.py
 
 [report]
 show_missing = True
+exclude_lines =
+    # re-enable the standard pragma
+    pragma: nocover
+    pragma: no cover
+    # exclude nvml calls
+    [\s(.]nvml[\s(.]
+    [\s(.]pynvml[\s(.]
+    # exclude LOG_PDB
+    LOG_PDB
+    # always ignore type checking blocks
+    TYPE_CHECKING
+    except ImportError
 
 [html]
 directory = coverage_html_report

.git-blame-ignore-revs

@@ -0,0 +1,33 @@
+# Auto flake and pep8 (#1353)
+7d252089a27ede17e6c78cecde485f651c1a7bdb
+
+# Apply Black to standardize code styling (#2614)
+04ae4212cf6d1e513a4ea83666719c79b7e5867a
+
+# Update black (#2901)
+741ffb60b94b15d2f243fc4ad4a849df76c46092
+
+# Fixup black string normalization (#2929)
+cf10db7b6a4fd091c2e1385162e3d36ab59c8f6e
+
+726f65438815317bd6c430b983463cfdbe34712b
+# Use latest release of black (#3388)
+726f65438815317bd6c430b983463cfdbe34712b
+
+# Rerun `black` on the code base (#3444)
+9af811d8f9858c63b9586bcfb78ce2dec8f5d6b3
+
+# Update for black (#4081)
+44bf0b981039ef5e474fbc5ccc6cd5a98b42e5e4
+
+# Pin black pre-commit (#4533)
+fdeca218134dbfe6c2c46f947413c7d0e1d2acab
+
+# Add isort to pre-commit hooks, package resorting (#4647)
+20a55e91b52d8e51a62ba1b47ccc1ece07adb72e
+
+# Pyupgrade (#4741)
+5dc591bbdd4427fe49fe90338a34fc85ee35f2c9
+
+# Flake8 config cleanup (#4888)
+ee06a44cc4b43270b5ae6ee102481f3bbc9daf27

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`
+- [ ] Passes `pre-commit run --all-files`

.github/workflows/conda.yml

@@ -0,0 +1,79 @@
+name: Conda build
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - setup.py
+      - requirements.txt
+      - continuous_integration/recipes/**
+      - .github/workflows/conda.yml
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: conda-${{ github.head_ref }}
+  cancel-in-progress: true
+
+# Required shell entrypoint to have properly activated conda environments
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  conda:
+    name: Build (and upload)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          mamba install boa conda-verify
+
+          which python
+          pip list
+          mamba list
+      - name: Build conda packages
+        run: |
+          # suffix for pre-release package versions
+          export VERSION_SUFFIX=a`date +%y%m%d`
+
+          # conda search for the latest dask-core pre-release
+          arr=($(conda search --override-channels -c dask/label/dev dask-core | tail -n 1))
+
+          # extract dask-core pre-release version / build
+          export DASK_CORE_VERSION=${arr[1]}
+
+          # distributed pre-release build
+          conda mambabuild continuous_integration/recipes/distributed \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder .
+
+          # dask pre-release build
+          conda mambabuild continuous_integration/recipes/dask \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder .
+      - name: Upload conda packages
+        if: |
+          github.event_name == 'push'
+          && github.ref == 'refs/heads/main'
+          && github.repository == 'dask/distributed'
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
+        run: |
+          # install anaconda for upload
+          mamba install anaconda-client
+
+          anaconda upload --label dev noarch/*.tar.bz2
+          anaconda upload --label dev linux-64/*.tar.bz2

.github/workflows/publish-test-results.yaml

@@ -0,0 +1,45 @@
+# Copied from https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.23/README.md#support-fork-repositories-and-dependabot-branches
+# Warning: changes to this workflow will NOT be picked up until they land in the main branch!
+# See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
+
+name: Publish test results
+
+on:
+  workflow_run:
+    workflows: [Tests]
+    types: [completed]
+
+jobs:
+  publish-test-results:
+    name: Publish test results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    # Needed to post comments on the PR
+    permissions:
+      checks: write
+      pull-requests: write
+
+    steps:
+      - name: Download and extract artifacts
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir artifacts && cd artifacts
+
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: artifacts/**/*.xml

.github/workflows/ssh_debug.yaml → .github/workflows/ssh_debug.yaml.donotrun

@@ -1,7 +1,6 @@
 name: Debug passwordless `ssh localhost`
 
-on: []
-# on: [pull_request]  # Uncomment to enable
+on: [push, pull_request]
 
 jobs:
   test:

.github/workflows/test-report.yaml

@@ -0,0 +1,51 @@
+name: Test Report
+
+on:
+  schedule:
+    # Run 2h after the daily tests.yaml
+    - cron: "0 8,20 * * *"
+  workflow_dispatch:
+
+jobs:
+  test-report:
+    name: Test Report
+    # Do not run the report job on forks
+    if: github.repository == 'dask/distributed' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          condarc-file: continuous_integration/condarc
+          use-mamba: true
+          python-version: 3.9
+          environment-file: continuous_integration/scripts/test-report-environment.yml
+          activate-environment: dask-distributed
+
+      - name: Show conda options
+        shell: bash -l {0}
+        run: conda config --show
+
+      - name: mamba list
+        shell: bash -l {0}
+        run: mamba list
+
+      - name: Generate report
+        shell: bash -l {0}
+        run: |
+          python continuous_integration/scripts/test_report.py --days 90 --nfails 1 -o test_report.html
+          python continuous_integration/scripts/test_report.py --days 7 --nfails 2 -o test_short_report.html
+          mkdir deploy
+          mv test_report.html test_short_report.html deploy/
+
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@4.1.7
+        with:
+          branch: gh-pages
+          folder: deploy