Drop support for Python 3.9 (#8793)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>

.github/workflows/ci-pre-commit.yml

@@ -14,5 +14,5 @@ jobs:
       - uses: actions/checkout@v4.1.3
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.12'
       - uses: pre-commit/action@v3.0.1

.github/workflows/tests.yaml

@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        environment: [mindeps, "3.9", "3.10", "3.11", "3.12"]
+        environment: [mindeps, "3.10", "3.11", "3.12"]
         label: [default]
         extra_packages: [null]
         # Cherry-pick test modules to split the overall runtime roughly in half
@@ -36,8 +36,6 @@ jobs:
           # MacOS CI does not have any hosts available; run it on 3.12 only
           - os: macos-latest
             environment: mindeps
-          - os: macos-latest
-            environment: "3.9"
           - os: macos-latest
             environment: "3.10"
           - os: macos-latest
@@ -49,21 +47,21 @@ jobs:
         include:
           # Set distributed.scheduler.worker-saturation: .inf
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_queue
             partition: "ci1"
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_queue
             partition: "not ci1"
 
           # Set dataframe.query-planning: false
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_expr
             partition: "ci1"
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_expr
             partition: "not ci1"
 

continuous_integration/environment-3.10.yaml

@@ -11,6 +11,7 @@ dependencies:
   - click
   - cloudpickle
   - coverage
+  - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec  # overridden by git tip below
   - gilknocker
@@ -21,26 +22,31 @@ dependencies:
   - jupyter-server-proxy
   - jupyterlab
   - locket
+  - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
   - pre-commit
   - prometheus_client
   - psutil
   - pyarrow
+  - pynvml  # Only tested here
   - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
   - requests
   - s3fs  # overridden by git tip below
   - scikit-learn
   - scipy
   - sortedcollections
   - tblib
   - toolz
+  - torchvision  # Only tested here
   - tornado
   - zict  # overridden by git tip below
   - zstandard
@@ -50,6 +56,7 @@ dependencies:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask-contrib/dask-expr
       - git+https://github.com/dask/zict
+      - git+https://github.com/dask/crick  # Only tested here
       # Revert after https://github.com/dask/distributed/issues/8614 is fixed
       # - git+https://github.com/dask/s3fs
       # - git+https://github.com/fsspec/filesystem_spec

continuous_integration/environment-mindeps.yaml

@@ -1,23 +1,22 @@
 name: dask-distributed
 channels:
   - conda-forge
-  - defaults
 dependencies:
-  - python=3.9
+  - python=3.10
   - click=8.0
-  - cloudpickle=1.5.0
-  - cytoolz=0.10.1
+  - cloudpickle=2.0.0
+  - cytoolz=0.11.2
   - jinja2=2.10.3
   - locket=1.0.0
-  - msgpack-python=1.0.0
+  - msgpack-python=1.0.2
   - packaging=20.0
-  - psutil=5.7.2
-  - pyyaml=5.3.1
+  - psutil=5.8.0
+  - pyyaml=5.4.1
   - sortedcontainers=2.0.5
   - tblib=1.6.0
   - toolz=0.10.0
-  - tornado=6.0.4
-  - urllib3=1.24.3
+  - tornado=6.2.0
+  - urllib3=1.26.5
   - zict=3.0.0
   # Temporary fix for https://github.com/pypa/setuptools/issues/4496
   - setuptools < 71

continuous_integration/gpuci/axis.yaml

@@ -1,5 +1,4 @@
 PYTHON_VER:
-- "3.9"
 - "3.10"
 - "3.11"
 

continuous_integration/recipes/dask/meta.yaml

@@ -19,19 +19,19 @@ build:
 
 requirements:
   host:
-    - python >=3.9
+    - python >=3.10
     - dask-core {{ dask_version }}
     - dask-expr {{ dask_expr_version }}
     - distributed {{ version }}
   run:
-    - python >=3.9
+    - python >=3.10
     - {{ pin_compatible('dask-core', max_pin='x.x.x.x') }}
     - {{ pin_compatible('dask-expr', max_pin='x.x.x.x') }}
     - {{ pin_compatible('distributed', exact=True) }}
-    - cytoolz >=0.8.2
+    - cytoolz >=0.11.2
     - lz4 >=4.3.2
     - numpy >=1.21
-    - pandas >=1.3
+    - pandas >=2
     - bokeh >=2.4.2,!=3.0.*
     - jinja2 >=2.10.3
     - pyarrow >=7.0

continuous_integration/recipes/distributed/meta.yaml

@@ -24,28 +24,28 @@ build:
 
 requirements:
   host:
-    - python >=3.9
+    - python >=3.10
     - pip
     - dask-core {{ dask_version }}
     - versioneer =0.29
     - tomli # [py<311]
   run:
-    - python >=3.9
+    - python >=3.10
     - click >=8.0
-    - cloudpickle >=1.5.0
-    - cytoolz >=0.10.1
+    - cloudpickle >=2.0.0
+    - cytoolz >=0.11.2
     - {{ pin_compatible('dask-core', max_pin='x.x.x.x') }}
     - jinja2 >=2.10.3
     - locket >=1.0.0
-    - msgpack-python >=1.0.0
+    - msgpack-python >=1.0.2
     - packaging >=20.0
-    - psutil >=5.7.2
-    - pyyaml >=5.3.1
+    - psutil >=5.8.0
+    - pyyaml >=5.4.1
     - sortedcontainers >=2.0.5
     - tblib >=1.6.0
     - toolz >=0.10.0
-    - tornado >=6.0.4
-    - urllib3 >=1.24.3
+    - tornado >=6.2.0
+    - urllib3 >=1.26.5
     - zict >=3.0.0
   run_constrained:
     - openssl !=1.1.1e

distributed/comm/registry.py

@@ -15,7 +15,7 @@ def __call__(self, **kwargs: str) -> Iterable[importlib.metadata.EntryPoint]:
 if sys.version_info >= (3, 10):
     # py3.10 importlib.metadata type annotations are not in mypy yet
     # https://github.com/python/typeshed/pull/7331
-    _entry_points: _EntryPoints = importlib.metadata.entry_points
+    _entry_points: _EntryPoints = importlib.metadata.entry_points  # type: ignore[assignment]
 else:
 
     def _entry_points(

distributed/protocol/pickle.py

@@ -6,12 +6,9 @@
 import pickle
 
 import cloudpickle
-from packaging.version import parse as parse_version
 
 from distributed.protocol.serialize import dask_deserialize, dask_serialize
 
-CLOUDPICKLE_GE_20 = parse_version(cloudpickle.__version__) >= parse_version("2.0.0")
-
 HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
 
 logger = logging.getLogger(__name__)
@@ -68,8 +65,7 @@ def dumps(x, *, buffer_callback=None, protocol=HIGHEST_PROTOCOL):
             pickler.dump(x)
             result = f.getvalue()
         if b"__main__" in result or (
-            CLOUDPICKLE_GE_20
-            and getattr(inspect.getmodule(x), "__name__", None)
+            getattr(inspect.getmodule(x), "__name__", None)
             in cloudpickle.list_registry_pickle_by_value()
         ):
             if len(result) < 1000 or not _always_use_pickle_for(x):

distributed/protocol/tests/test_pickle.py

@@ -13,12 +13,7 @@
 
 from distributed import profile
 from distributed.protocol import deserialize, serialize
-from distributed.protocol.pickle import (
-    CLOUDPICKLE_GE_20,
-    HIGHEST_PROTOCOL,
-    dumps,
-    loads,
-)
+from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 from distributed.protocol.serialize import dask_deserialize, dask_serialize
 from distributed.utils_test import popen, save_sys_modules
 
@@ -200,9 +195,6 @@ def funcs():
             assert wr3() is None
 
 
-@pytest.mark.skipif(
-    not CLOUDPICKLE_GE_20, reason="Pickle by value registration not supported"
-)
 def test_pickle_by_value_when_registered():
     with save_sys_modules():
         with tmpdir() as d:

pyproject.toml

@@ -16,30 +16,29 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering",
     "Topic :: System :: Distributed Computing",
 ]
 readme = "README.rst"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "click >= 8.0",
-    "cloudpickle >= 1.5.0",
+    "cloudpickle >= 2.0.0",
     "dask == 2024.8.0",
     "jinja2 >= 2.10.3",
     "locket >= 1.0.0",
-    "msgpack >= 1.0.0",
+    "msgpack >= 1.0.2",
     "packaging >= 20.0",
-    "psutil >= 5.7.2",
-    "pyyaml >= 5.3.1",
+    "psutil >= 5.8.0",
+    "pyyaml >= 5.4.1",
     "sortedcontainers >= 2.0.5",
     "tblib >= 1.6.0",
-    "toolz >= 0.10.0",
-    "tornado >= 6.0.4",
-    "urllib3 >= 1.24.3",
+    "toolz >= 0.11.2",
+    "tornado >= 6.2.0",
+    "urllib3 >= 1.26.5",
     "zict >= 3.0.0",
 ]
 dynamic = ["version"]
@@ -178,7 +177,7 @@ timeout_method = "thread"
 timeout = 300
 
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.10"
 # See https://github.com/python/mypy/issues/12286 for automatic multi-platform support
 platform = "linux"
 # platform = win32