diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index b1dfbc9ce1..be6711f890 100644 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -71,9 +71,11 @@ if [[ $PYTHON == 3.6 ]]; then conda install -c conda-forge -c defaults contextvars fi -# stacktrace is not currently avaiable for Python 3.8. -# Remove the version check block below when it is avaiable. if [[ $PYTHON != 3.8 ]]; then + # Install backport package for pickle protocol 5 support + conda install -c conda-forge -c defaults 'pickle5>=0.0.11' + # stacktrace is not currently avaiable for Python 3.8. + # Remove the version check block below when it is avaiable. # For low-level profiler, install libunwind and stacktrace from conda-forge # For stacktrace we use --no-deps to avoid upgrade of python conda install -c conda-forge -c defaults libunwind diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py index 6e168947d4..fd2343756a 100644 --- a/distributed/protocol/pickle.py +++ b/distributed/protocol/pickle.py @@ -1,8 +1,16 @@ import logging -import pickle +import sys import cloudpickle +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index 9ee496f5e9..ea2143c535 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -1,7 +1,6 @@ from functools import partial import gc from operator import add -import pickle import weakref import sys @@ -10,6 +9,14 @@ from distributed.protocol import deserialize, serialize from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + def test_pickle_data(): data = [1, b"123", "123", [123], {}, set()] diff --git a/distributed/scheduler.py b/distributed/scheduler.py index 4d22a320e5..f96385656e 100644 --- a/distributed/scheduler.py +++ b/distributed/scheduler.py @@ -12,7 +12,7 @@ from numbers import Number import operator import os -import pickle +import sys import random import warnings import weakref @@ -86,6 +86,14 @@ from .stealing import WorkStealing from .variable import VariableExtension +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + logger = logging.getLogger(__name__) diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py index d6e6714513..7537d0d63e 100644 --- a/distributed/tests/test_scheduler.py +++ b/distributed/tests/test_scheduler.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import pickle import operator import re import sys @@ -42,6 +41,14 @@ from distributed.utils_test import loop, nodebug # noqa: F401 from dask.compatibility import apply +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + alice = "alice:1234" bob = "bob:1234" diff --git a/requirements.txt b/requirements.txt index 95a681d66c..516ceb62a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ click >= 6.6 -cloudpickle >= 1.3.0 +cloudpickle >= 1.5.0 contextvars;python_version<'3.7' dask >= 2.9.0 msgpack >= 0.6.0