Add support for Python 3.12 (#972)

* Add support for Python 3.12

* Use pyproject.toml to make setuptools pick up the driver version

* Add python 3.12 tag to package metadata

* Comment why wait_for shim not needed in Python 3.12

* Silence deprecation warnings for Python 3.12 for now

---------

Signed-off-by: Rouven Bauer <rouven.bauer@neo4j.com>
Signed-off-by: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com>
Co-authored-by: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com>

docs/source/index.rst

@@ -12,6 +12,7 @@ Neo4j versions supported:
 
 Python versions supported:
 
+* Python 3.12 (added in driver version 5.14.0)
 * Python 3.11 (added in driver version 5.3.0)
 * Python 3.10
 * Python 3.9

pyproject.toml

@@ -38,6 +38,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 dynamic = ["version", "readme"]
 
@@ -50,17 +51,19 @@ pandas = [
     "pandas >= 1.1.0, < 3.0.0",
     "numpy >= 1.7.0, < 2.0.0",
 ]
+pyarrow = ["pyarrow >= 1.0.0"]
+
 
 [build-system]
 requires = [
-    "setuptools~=65.6",
-    "tomlkit~=0.11.6",
+    "setuptools >= 66.1.0",
+    # TODO: 6.0 - can be removed once `setup.py` is simplified
+    "tomlkit ~= 0.11.6",
 ]
 build-backend = "setuptools.build_meta"
 
-# still in beta
-#[tool.setuptools.dynamic]
-#version = {attr = "neo4j._meta.version"}
+[tool.setuptools.dynamic]
+version = {attr = "neo4j._meta.version"}
 
 
 [tool.coverage]

requirements-dev.txt

@@ -1,5 +1,5 @@
 # the driver itself
--e .[pandas,numpy]
+-e .[pandas,numpy,pyarrow]
 
 # auto-generate sync driver from async code
 unasync>=0.5.0
@@ -10,16 +10,10 @@ mypy>=0.971
 typing-extensions>=4.3.0
 types-pytz>=2022.1.2
 
-# for packaging
-setuptools~=65.6
-# TODO: 6.0 - can be removed once `setup.py` is simplified
-tomlkit~=0.11.6
-
 # needed for running tests
 coverage[toml]>=5.5
 freezegun >= 1.2.2
 mock>=4.0.3
-pyarrow>=1.0.0
 pytest>=6.2.5
 pytest-asyncio>=0.16.0
 pytest-benchmark>=3.4.1

setup.py

@@ -35,7 +35,6 @@
 from neo4j._meta import (
     deprecated_package as deprecated,
     package,
-    version,
 )
 
 
@@ -84,8 +83,4 @@ def changed_package_name(new_name):
 
 
 with changed_package_name(package):
-    setup(
-        # until `[tool.setuptools.dynamic]` in pyproject.toml is out of beta
-        version=version,
-        long_description=readme,
-    )
+    setup(long_description=readme)

src/neo4j/_async_compat/concurrency.py

@@ -136,6 +136,8 @@ async def acquire(self, blocking=True, timeout=-1):
             try:
                 await wait_for(fut, timeout)
             except asyncio.CancelledError:
+                if fut.cancelled():
+                    raise
                 already_finished = not fut.cancel()
                 if already_finished:
                     # Too late to cancel the acquisition.

src/neo4j/_async_compat/shims/__init__.py

@@ -25,12 +25,11 @@
 # The shipped wait_for can swallow cancellation errors (starting with 3.8).
 # See: https://github.com/python/cpython/pull/26097
 # and https://github.com/python/cpython/pull/28149
-# Since 3.8 and 3.9 already received their final maintenance release, there
-# will be now fix for this. So this patch needs to stick around at least until
-# we remove support for Python 3.9.
+# Ultimately, this got fixed in https://github.com/python/cpython/pull/98518
+# (released with Python 3.12) by re-doing how wait_for works.
 
 
-if sys.version_info >= (3, 8):
+if (3, 12) > sys.version_info >= (3, 8):
     # copied from Python 3.10's asyncio package with applied patch
 
     # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,

testkit/Dockerfile

@@ -42,19 +42,29 @@ ENV PYENV_ROOT /.pyenv
 ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
 
 # Setup python version
-ENV PYTHON_VERSIONS 3.7 3.8 3.9 3.10 3.11
+ENV PYTHON_VERSIONS 3.12 3.11 3.10 3.9 3.8 3.7
 
 RUN for version in $PYTHON_VERSIONS; do \
         pyenv install $version; \
     done
 RUN pyenv rehash
-RUN pyenv global $(pyenv versions --bare --skip-aliases)
+RUN pyenv global $(pyenv versions --bare --skip-aliases | sort --version-sort --reverse)
 
 # Install Latest pip and setuptools for each environment
 # + tox and tools for starting the tests
 # https://pip.pypa.io/en/stable/news/
 RUN for version in $PYTHON_VERSIONS; do \
         python$version -m pip install -U pip && \
-        python$version -m pip install -U setuptools && \
         python$version -m pip install -U coverage tox; \
     done
+
+# Installing pyarrow lib until pre-built wheel for Python 3.12 exists
+# https://github.com/apache/arrow/issues/37880
+RUN apt update && \
+    apt install -y -V lsb-release cmake gcc && \
+    distro_name=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \
+    code_name=$(lsb_release --codename --short) && \
+    wget https://apache.jfrog.io/artifactory/arrow/${distro_name}/apache-arrow-apt-source-latest-${code_name}.deb && \
+    apt install -y -V ./apache-arrow-apt-source-latest-${code_name}.deb && \
+    apt update && \
+    apt install -y -V libarrow-dev # For C++ \

testkit/_common.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import sys
 
@@ -7,14 +8,28 @@
 
 
 def run(args, env=None):
+    print(args)
     return subprocess.run(
         args, universal_newlines=True, stdout=sys.stdout, stderr=sys.stderr,
         check=True, env=env
     )
 
 
+def get_python_version():
+    cmd = [TEST_BACKEND_VERSION, "-V"]
+    res = subprocess.check_output(cmd, universal_newlines=True,
+                                  stderr=sys.stderr)
+    raw_version = re.match(r"(?:.*?)((?:\d+\.)+(?:\d+))", res).group(1)
+    return tuple(int(e) for e in raw_version.split("."))
+
+
 def run_python(args, env=None, warning_as_error=True):
     cmd = [TEST_BACKEND_VERSION, "-u"]
+    if get_python_version() >= (3, 12):
+        # Ignore warnings for Python 3.12 for now
+        # https://github.com/dateutil/dateutil/issues/1284 needs to be released
+        # and propagate through our dependency graph
+        warning_as_error = False
     if warning_as_error:
         cmd += ["-W", "error"]
     cmd += list(args)

testkitbackend/__main__.py

@@ -49,7 +49,11 @@ async def main():
 
 
 if __name__ == "__main__":
-    warnings.simplefilter("error")
+    if sys.version_info < (3, 12):
+        # Ignore warnings for Python 3.12 for now
+        # https://github.com/dateutil/dateutil/issues/1284 needs to be released
+        # and propagate through our dependency graph
+        warnings.simplefilter("error")
     if len(sys.argv) == 2 and sys.argv[1].lower().strip() == "async":
         async_main()
     else:

tests/unit/async_/test_driver.py

@@ -716,6 +716,7 @@ async def test_execute_query_keyword_parameters(
 @pytest.mark.parametrize("parameters", (
     {"_": "a"}, {"foo_": None}, {"foo_": 1, "bar_": 2}
 ))
+@mark_async_test
 async def test_reserved_query_keyword_parameters(
     mocker, parameters: t.Dict[str, t.Any],
 ) -> None:

tests/unit/async_/work/test_result.py

@@ -607,7 +607,7 @@ async def test_data(num_records):
         record.data.return_value = expected_data[-1]
     assert await result.data("hello", "world") == expected_data
     for record in records:
-        assert record.data.called_once_with("hello", "world")
+        record.data.assert_called_once_with("hello", "world")
 
 
 @pytest.mark.parametrize("records", (

tests/unit/mixed/async_compat/test_concurrency.py

@@ -225,7 +225,6 @@ async def waiter_non_blocking():
         assert fut.exception() is exc
         awaits += 1
 
-
     assert not lock.locked()
     await asyncio.gather(blocker(), waiter_non_blocking())
     assert not lock.locked()

tests/unit/mixed/async_compat/test_shims.py

@@ -42,8 +42,8 @@ async def _check_wait_for(wait_for_, should_propagate_cancellation):
 
 
 @pytest.mark.skipif(
-    sys.version_info < (3, 8),
-    reason="wait_for is only broken in Python 3.8+"
+    not (3, 12) > sys.version_info >= (3, 8),
+    reason="wait_for is only broken in Python 3.8-3.11 (inclusive)"
 )
 @mark_async_test
 async def test_wait_for_shim_is_necessary_starting_from_3x8():
@@ -56,8 +56,8 @@ async def test_wait_for_shim_is_necessary_starting_from_3x8():
 
 
 @pytest.mark.skipif(
-    sys.version_info >= (3, 8),
-    reason="wait_for is only broken in Python 3.8+"
+    (3, 12) > sys.version_info >= (3, 8),
+    reason="wait_for is only broken in Python 3.8-3.11 (inclusive)"
 )
 @mark_async_test
 async def test_wait_for_shim_is_not_necessary_prior_to_3x8():

tests/unit/sync/test_driver.py

@@ -715,6 +715,7 @@ def test_execute_query_keyword_parameters(
 @pytest.mark.parametrize("parameters", (
     {"_": "a"}, {"foo_": None}, {"foo_": 1, "bar_": 2}
 ))
+@mark_sync_test
 def test_reserved_query_keyword_parameters(
     mocker, parameters: t.Dict[str, t.Any],
 ) -> None:

tests/unit/sync/work/test_result.py

@@ -607,7 +607,7 @@ def test_data(num_records):
         record.data.return_value = expected_data[-1]
     assert result.data("hello", "world") == expected_data
     for record in records:
-        assert record.data.called_once_with("hello", "world")
+        record.data.assert_called_once_with("hello", "world")
 
 
 @pytest.mark.parametrize("records", (

tox.ini

@@ -1,15 +1,20 @@
 [tox]
-envlist = py{37,38,39,310,311}-{unit,integration,performance}
+envlist = py{37,38,39,310,311,312}-{unit,integration,performance}
 
 [testenv]
 passenv = TEST_NEO4J_*
 deps = -r requirements-dev.txt
 setenv = COVERAGE_FILE={envdir}/.coverage
 usedevelop = true
+# Ignore warnings for Python 3.12 for now
+# https://github.com/dateutil/dateutil/issues/1284 needs to be released
+# and propagate through our dependency graph
+warnargs =
+    py{37,38,39,310,311}: -W error
+    py312:
 commands =
     coverage erase
-    unit: coverage run -m pytest -W error -v {posargs} tests/unit
-    unit: coverage run -m pytest -v --doctest-modules {posargs} src
-    integration: coverage run -m pytest -W error -v {posargs} tests/integration
+    unit: coverage run -m pytest {[testenv]warnargs} -v {posargs} tests/unit
+    integration: coverage run -m pytest {[testenv]warnargs} -v {posargs} tests/integration
     performance: python -m pytest --benchmark-autosave -v {posargs} tests/performance
     unit,integration: coverage report