Merge pull request #1067 from nolar/python-3.12

Enable Python 3.12 in CI and as the main target

.github/workflows/ci.yaml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: pip install -r requirements.txt
       - run: pre-commit run --all-files
       - run: mypy kopf --strict
@@ -38,10 +38,10 @@ jobs:
       fail-fast: false
       matrix:
         install-extras: [ "", "full-auth" ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         include:
           - install-extras: "uvloop"
-            python-version: "3.11"
+            python-version: "3.12"
     name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }}
     runs-on: ubuntu-22.04
     timeout-minutes: 5  # usually 2-3 mins
@@ -111,7 +111,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: nolar/setup-k3d-k3s@v1
         with:
           version: ${{ matrix.k3s }}

.github/workflows/publish.yaml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: pip install --upgrade setuptools wheel twine
       - run: python setup.py sdist bdist_wheel
       - uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/thorough.yaml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: pip install -r requirements.txt
       - run: pre-commit run --all-files
       - run: mypy kopf --strict
@@ -42,10 +42,10 @@ jobs:
       fail-fast: false
       matrix:
         install-extras: [ "", "full-auth" ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         include:
           - install-extras: "uvloop"
-            python-version: "3.11"
+            python-version: "3.12"
     name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }}
     runs-on: ubuntu-22.04
     timeout-minutes: 5  # usually 2-3 mins
@@ -115,7 +115,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: nolar/setup-k3d-k3s@v1
         with:
           version: ${{ matrix.k3s }}
@@ -137,7 +137,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: tools/install-minikube.sh
       - run: pip install -r requirements.txt -r examples/requirements.txt
       - run: pytest --color=yes --timeout=30 --only-e2e

README.md

@@ -133,7 +133,7 @@ We assume that when the operator is executed in the cluster, it must be packaged
 into a docker image with a CI/CD tool of your preference.
 
 ```dockerfile
-FROM python:3.11
+FROM python:3.12
 ADD . /src
 RUN pip install kopf
 CMD kopf run /src/handlers.py --verbose

docs/deployment.rst

@@ -16,7 +16,7 @@ First of all, the operator must be packaged as a docker image with Python 3.8 or
     :caption: Dockerfile
     :name: dockerfile
 
-    FROM python:3.11
+    FROM python:3.12
     RUN pip install kopf
     ADD . /src
     CMD kopf run /src/handlers.py --verbose

kopf/_cogs/helpers/versions.py

@@ -13,13 +13,12 @@
 version: Optional[str] = None
 
 try:
-    import pkg_resources
+    import importlib.metadata
 except ImportError:
     pass
 else:
     try:
         name, *_ = __name__.split('.')  # usually "kopf", unless renamed/forked.
-        dist: pkg_resources.Distribution = pkg_resources.get_distribution(name)
-        version = dist.version
+        version = importlib.metadata.version(name)
     except Exception:
         pass  # installed as an egg, from git, etc.

kopf/_core/actions/loggers.py

@@ -29,7 +29,7 @@ class LogFormat(enum.Enum):
     """ Log formats, as specified on CLI. """
     PLAIN = '%(message)s'
     FULL = '[%(asctime)s] %(name)-20.20s [%(levelname)-8.8s] %(message)s'
-    JSON = enum.auto()
+    JSON = '-json-'  # not used for formatting, only for detection
 
 
 class ObjectFormatter(logging.Formatter):

kopf/_core/engines/posting.py

@@ -192,13 +192,14 @@ def createLock(self) -> None:
     def filter(self, record: logging.LogRecord) -> bool:
         # Only those which have a k8s object referred (see: `ObjectLogger`).
         # Otherwise, we have nothing to post, and nothing to do.
+        # TODO: remove all bool() -- they were needed for Python 3.12 & MyPy 1.8.0 wrong inference.
         settings: Optional[configuration.OperatorSettings]
         settings = getattr(record, 'settings', None)
-        level_ok = settings is not None and record.levelno >= settings.posting.level
-        enabled = settings is not None and settings.posting.enabled
+        level_ok = settings is not None and bool(record.levelno >= settings.posting.level)
+        enabled = settings is not None and bool(settings.posting.enabled)
         has_ref = hasattr(record, 'k8s_ref')
-        skipped = hasattr(record, 'k8s_skip') and getattr(record, 'k8s_skip')
-        return enabled and level_ok and has_ref and not skipped and super().filter(record)
+        skipped = hasattr(record, 'k8s_skip') and bool(getattr(record, 'k8s_skip'))
+        return enabled and level_ok and has_ref and not skipped and bool(super().filter(record))
 
     def emit(self, record: logging.LogRecord) -> None:
         # Same try-except as in e.g. `logging.StreamHandler`.

requirements.txt

@@ -17,7 +17,7 @@ pre-commit
 pyngrok
 pytest>=6.0.0
 pytest-aiohttp
-pytest-asyncio<0.22  # until the "event_loop" deprecation is solved
+pytest-asyncio
 pytest-cov
 pytest-mock
 pytest-timeout

setup.py

@@ -37,6 +37,7 @@
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
@@ -61,7 +62,8 @@
         'python-json-logger',   # 0.05 MB
         'iso8601',              # 0.07 MB
         'click',                # 0.60 MB
-        'aiohttp<4.0.0',        # 7.80 MB
+        'aiohttp',              # 7.80 MB
+        'aiohttp>=3.9.0; python_version>="3.12"',
         'pyyaml',               # 0.90 MB
     ],
     extras_require={
@@ -71,6 +73,7 @@
         ],
         'uvloop': [
             'uvloop',           # 9.00 MB
+            'uvloop>=0.18.0; python_version>="3.12"',
         ],
         'dev': [
             'pyngrok',          # 1.00 MB + downloaded binary

tests/conftest.py

@@ -38,6 +38,11 @@ def pytest_configure(config):
     # TODO: Remove when fixed in https://github.com/pytest-dev/pytest-asyncio/issues/460:
     config.addinivalue_line('filterwarnings', 'ignore:There is no current event loop:DeprecationWarning:pytest_asyncio')
 
+    # Python 3.12 transitional period:
+    config.addinivalue_line('filterwarnings', 'ignore:datetime*:DeprecationWarning:dateutil')
+    config.addinivalue_line('filterwarnings', 'ignore:datetime*:DeprecationWarning:freezegun')
+    config.addinivalue_line('filterwarnings', 'ignore:.*:DeprecationWarning:_pydevd_.*')
+
 
 def pytest_addoption(parser):
     parser.addoption("--only-e2e", action="store_true", help="Execute end-to-end tests only.")
@@ -698,8 +703,13 @@ def assert_logs_fn(patterns, prohibited=[], strict=False):
 #
 # Helpers for asyncio checks.
 #
+@pytest.fixture()
+async def loop():
+    yield asyncio.get_running_loop()
+
+
 @pytest.fixture(autouse=True)
-def _no_asyncio_pending_tasks(event_loop):
+def _no_asyncio_pending_tasks(loop: asyncio.AbstractEventLoop):
     """
     Ensure there are no unattended asyncio tasks after the test.
 
@@ -720,7 +730,7 @@ def _no_asyncio_pending_tasks(event_loop):
 
     # Let the pytest-asyncio's async2sync wrapper to finish all callbacks. Otherwise, it raises:
     #   <Task pending name='Task-2' coro=<<async_generator_athrow without __name__>()>>
-    event_loop.run_until_complete(asyncio.sleep(0))
+    loop.run_until_complete(asyncio.sleep(0))
 
     # Detect all leftover tasks.
     after = _get_all_tasks()
@@ -734,7 +744,10 @@ def _get_all_tasks() -> Set[asyncio.Task]:
     i = 0
     while True:
         try:
-            tasks = list(asyncio.tasks._all_tasks)
+            if sys.version_info >= (3, 12):
+                tasks = asyncio.tasks._eager_tasks | set(asyncio.tasks._scheduled_tasks)
+            else:
+                tasks = list(asyncio.tasks._all_tasks)
         except RuntimeError:
             i += 1
             if i >= 1000:

tests/logging/conftest.py

@@ -11,10 +11,10 @@ def _caplog_all_levels(caplog):
 
 
 @pytest.fixture(autouse=True)
-def event_queue_loop(event_loop):
-    token = event_queue_loop_var.set(event_loop)
+def event_queue_loop(loop):  # must be sync-def
+    token = event_queue_loop_var.set(loop)
     try:
-        yield event_loop
+        yield loop
     finally:
         event_queue_loop_var.reset(token)
 

tests/posting/conftest.py

@@ -6,10 +6,10 @@
 
 
 @pytest.fixture()
-def event_queue_loop(event_loop):
-    token = event_queue_loop_var.set(event_loop)
+def event_queue_loop(loop):  # must be sync-def
+    token = event_queue_loop_var.set(loop)
     try:
-        yield event_loop
+        yield loop
     finally:
         event_queue_loop_var.reset(token)
 

tests/posting/test_threadsafety.py

@@ -49,14 +49,14 @@
 
 
 @pytest.fixture()
-def awakener(event_loop):
+def awakener():
     handles = []
 
     def noop():
         pass
 
     def awaken_fn(delay, fn=noop):
-        handle = event_loop.call_later(delay, fn)
+        handle = asyncio.get_running_loop().call_later(delay, fn)
         handles.append(handle)
 
     try:

tests/primitives/test_conditions.py

@@ -18,7 +18,7 @@ async def test_no_triggering():
         await asyncio.wait([task])
 
 
-async def test_triggering(event_loop, timer):
+async def test_triggering(timer):
     source = asyncio.Condition()
     target = asyncio.Condition()
     task = asyncio.create_task(condition_chain(source, target))
@@ -28,7 +28,8 @@ async def delayed_trigger():
             async with source:
                 source.notify_all()
 
-        event_loop.call_later(0.1, asyncio.create_task, delayed_trigger())
+        loop = asyncio.get_running_loop()
+        loop.call_later(0.1, asyncio.create_task, delayed_trigger())
 
         with timer:
             async with target:

tests/primitives/test_containers.py

@@ -11,19 +11,20 @@ async def test_empty_by_default():
         await asyncio.wait_for(container.wait(), timeout=0.1)
 
 
-async def test_does_not_wake_up_when_reset(event_loop, timer):
+async def test_does_not_wake_up_when_reset(timer):
     container = Container()
 
     async def reset_it():
         await container.reset()
 
-    event_loop.call_later(0.05, asyncio.create_task, reset_it())
+    loop = asyncio.get_running_loop()
+    loop.call_later(0.05, asyncio.create_task, reset_it())
 
     with pytest.raises(asyncio.TimeoutError):
         await asyncio.wait_for(container.wait(), timeout=0.1)
 
 
-async def test_wakes_up_when_preset(event_loop, timer):
+async def test_wakes_up_when_preset(timer):
     container = Container()
     await container.set(123)
 
@@ -34,13 +35,14 @@ async def test_wakes_up_when_preset(event_loop, timer):
     assert result == 123
 
 
-async def test_wakes_up_when_set(event_loop, timer):
+async def test_wakes_up_when_set(timer):
     container = Container()
 
     async def set_it():
         await container.set(123)
 
-    event_loop.call_later(0.1, asyncio.create_task, set_it())
+    loop = asyncio.get_running_loop()
+    loop.call_later(0.1, asyncio.create_task, set_it())
 
     with timer:
         result = await container.wait()
@@ -49,14 +51,15 @@ async def set_it():
     assert result == 123
 
 
-async def test_iterates_when_set(event_loop, timer):
+async def test_iterates_when_set(timer):
     container = Container()
 
     async def set_it(v):
         await container.set(v)
 
-    event_loop.call_later(0.1, asyncio.create_task, set_it(123))
-    event_loop.call_later(0.2, asyncio.create_task, set_it(234))
+    loop = asyncio.get_running_loop()
+    loop.call_later(0.1, asyncio.create_task, set_it(123))
+    loop.call_later(0.2, asyncio.create_task, set_it(234))
 
     values = []
     with timer:
@@ -69,7 +72,7 @@ async def set_it(v):
     assert values == [123, 234]
 
 
-async def test_iterates_when_preset(event_loop, timer):
+async def test_iterates_when_preset(timer):
     container = Container()
     await container.set(123)
 

tests/reactor/conftest.py

@@ -41,7 +41,7 @@ def watcher_limited(mocker, settings):
 
 
 @pytest.fixture()
-def watcher_in_background(settings, resource, event_loop, worker_spy, stream):
+async def watcher_in_background(settings, resource, worker_spy, stream):
 
     # Prevent remembering the streaming objects in the mocks.
     async def do_nothing(*args, **kwargs):
@@ -57,7 +57,7 @@ async def do_nothing(*args, **kwargs):
         settings=settings,
         processor=do_nothing,
     )
-    task = event_loop.create_task(coro)
+    task = asyncio.create_task(coro)
 
     try:
         # Go for a test.
@@ -66,6 +66,6 @@ async def do_nothing(*args, **kwargs):
         # Terminate the watcher to cleanup the loop.
         task.cancel()
         try:
-            event_loop.run_until_complete(task)
+            await task
         except asyncio.CancelledError:
             pass  # cancellations are expected at this point

tests/reactor/test_queueing.py

@@ -123,7 +123,7 @@ async def test_watchevent_demultiplexing(worker_mock, timer, resource, processor
 ])
 @pytest.mark.usefixtures('watcher_limited')
 async def test_watchevent_batching(settings, resource, processor, timer,
-                                   stream, events, uids, vals, event_loop):
+                                   stream, events, uids, vals):
     """ Verify that only the last event per uid is actually handled. """
 
     # Override the default timeouts to make the tests faster.