Optionally use pickle5 (#3849)

Optionally use pickle5

continuous_integration/travis/install.sh

@@ -71,9 +71,11 @@ if [[ $PYTHON == 3.6 ]]; then
   conda install -c conda-forge -c defaults contextvars
 fi
 
-# stacktrace is not currently avaiable for Python 3.8.
-# Remove the version check block below when it is avaiable.
 if [[ $PYTHON != 3.8 ]]; then
+    # Install backport package for pickle protocol 5 support
+    conda install -c conda-forge -c defaults 'pickle5>=0.0.11'
+    # stacktrace is not currently avaiable for Python 3.8.
+    # Remove the version check block below when it is avaiable.
     # For low-level profiler, install libunwind and stacktrace from conda-forge
     # For stacktrace we use --no-deps to avoid upgrade of python
     conda install -c conda-forge -c defaults libunwind

distributed/protocol/pickle.py

@@ -1,8 +1,16 @@
 import logging
-import pickle
+import sys
 
 import cloudpickle
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
 

distributed/protocol/tests/test_pickle.py

@@ -1,7 +1,6 @@
 from functools import partial
 import gc
 from operator import add
-import pickle
 import weakref
 import sys
 
@@ -10,6 +9,14 @@
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 def test_pickle_data():
     data = [1, b"123", "123", [123], {}, set()]

distributed/scheduler.py

@@ -12,7 +12,7 @@
 from numbers import Number
 import operator
 import os
-import pickle
+import sys
 import random
 import warnings
 import weakref
@@ -86,6 +86,14 @@
 from .stealing import WorkStealing
 from .variable import VariableExtension
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 logger = logging.getLogger(__name__)
 

distributed/tests/test_scheduler.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import logging
-import pickle
 import operator
 import re
 import sys
@@ -42,6 +41,14 @@
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from dask.compatibility import apply
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 alice = "alice:1234"
 bob = "bob:1234"

requirements.txt

@@ -1,5 +1,5 @@
 click >= 6.6
-cloudpickle >= 1.3.0
+cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
 dask >= 2.9.0
 msgpack >= 0.6.0