diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fff9b40fdd..f41494303f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,9 @@ jobs: tox_env: py38-full - python: '3.9' tox_env: py39-full - - python: 'pypy3' + - python: '3.10' + tox_env: py310-full + - python: 'pypy-3.8' # Pypy is a lot slower due to jit warmup costs, so don't run the # "full" test config there. tox_env: pypy3 diff --git a/docs/index.rst b/docs/index.rst index 6f59bd7085..7877be8c3b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -96,7 +96,7 @@ installed in this way, so you may wish to download a copy of the source tarball or clone the `git repository `_ as well. -**Prerequisites**: Tornado 6.0 requires Python 3.6 or newer (See +**Prerequisites**: Tornado 6.0 requires Python 3.7 or newer (See `Tornado 5.1 `_ if compatibility with Python 2.7 is required). The following optional packages may be useful: diff --git a/pyproject.toml b/pyproject.toml index 73200f4f44..29e9dab68a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.cibuildwheel] -build = "cp3[789]*" +build = "cp3[789]* cp310*" test-command = "python -m tornado.test" [tool.cibuildwheel.macos] diff --git a/setup.cfg b/setup.cfg index a365d2c33c..e8bf80679f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ license_file = LICENSE [mypy] -python_version = 3.6 +python_version = 3.7 no_implicit_optional = True [mypy-tornado.*,tornado.platform.*] diff --git a/setup.py b/setup.py index 7c9b35c5af..3f82314d04 100644 --- a/setup.py +++ b/setup.py @@ -138,7 +138,7 @@ def build_extension(self, ext): if setuptools is not None: - python_requires = ">= 3.6" + python_requires = ">= 3.7" kwargs["python_requires"] = python_requires setup( @@ -180,10 +180,10 @@ def build_extension(self, ext): classifiers=[ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], diff --git a/tornado/ioloop.py b/tornado/ioloop.py index 28d86f5049..eec767ec09 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -41,6 +41,7 @@ import time import math import random +import warnings from inspect import isawaitable from tornado.concurrent import ( @@ -86,6 +87,7 @@ class IOLoop(Configurable): .. testcode:: + import asyncio import errno import functools import socket @@ -108,7 +110,7 @@ def connection_ready(sock, fd, events): io_loop = tornado.ioloop.IOLoop.current() io_loop.spawn_callback(handle_connection, connection, address) - if __name__ == '__main__': + async def main(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.setblocking(0) @@ -118,18 +120,18 @@ def connection_ready(sock, fd, events): io_loop = tornado.ioloop.IOLoop.current() callback = functools.partial(connection_ready, sock) io_loop.add_handler(sock.fileno(), callback, io_loop.READ) - io_loop.start() + await asyncio.Event().wait() + + if __name__ == "__main__": + asyncio.run(main()) .. testoutput:: :hide: - By default, a newly-constructed `IOLoop` becomes the thread's current - `IOLoop`, unless there already is a current `IOLoop`. This behavior - can be controlled with the ``make_current`` argument to the `IOLoop` - constructor: if ``make_current=True``, the new `IOLoop` will always - try to become current and it raises an error if there is already a - current instance. If ``make_current=False``, the new `IOLoop` will - not try to become current. + Do not attempt to construct an `IOLoop` directly; this is deprecated + since Tornado 6.2. Instead, initialize the `asyncio` event loop and + use `IOLoop.current()` to access an `IOLoop` wrapper around the + current event loop. In general, an `IOLoop` cannot survive a fork or be shared across processes in any way. When multiple processes are being used, each @@ -151,6 +153,12 @@ def connection_ready(sock, fd, events): ``IOLoop.configure`` method cannot be used on Python 3 except to redundantly specify the `asyncio` event loop. + .. deprecated:: 6.2 + It is deprecated to create an event loop that is "current" but not + currently running. This means it is deprecated to pass + ``make_current=True`` to the ``IOLoop`` constructor, or to create + an ``IOLoop`` while no asyncio event loop is running unless + ``make_current=False`` is used. """ # These constants were originally based on constants from the epoll module. @@ -259,6 +267,10 @@ def current(instance: bool = True) -> Optional["IOLoop"]: # noqa: F811 an alias for this method). ``instance=False`` is deprecated, since even if we do not create an `IOLoop`, this method may initialize the asyncio loop. + + .. deprecated:: 6.2 + It is deprecated to call ``IOLoop.current()`` when no `asyncio` + event loop is running. """ try: loop = asyncio.get_event_loop() @@ -292,6 +304,13 @@ def make_current(self) -> None: .. versionchanged:: 5.0 This method also sets the current `asyncio` event loop. + + .. deprecated:: 6.2 + The concept of an event loop that is "current" without + currently running is deprecated in asyncio since Python + 3.10. All related functionality in Tornado is also + deprecated. Instead, start the event loop with `asyncio.run` + before interacting with it. """ # The asyncio event loops override this method. raise NotImplementedError() @@ -304,7 +323,9 @@ def clear_current() -> None: .. versionchanged:: 5.0 This method also clears the current `asyncio` event loop. + .. deprecated:: 6.2 """ + warnings.warn("clear_current is deprecated", DeprecationWarning) old = IOLoop.current(instance=False) if old is not None: old._clear_current_hook() diff --git a/tornado/iostream.py b/tornado/iostream.py index 930f73d61a..b9955daf86 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -1530,6 +1530,7 @@ def _handle_connect(self) -> None: self._ssl_options, server_hostname=self._server_hostname, do_handshake_on_connect=False, + server_side=False, ) self._add_io_state(old_state) diff --git a/tornado/netutil.py b/tornado/netutil.py index 15eea9704f..4a1135405a 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -559,7 +559,8 @@ def resolve( def ssl_options_to_context( - ssl_options: Union[Dict[str, Any], ssl.SSLContext] + ssl_options: Union[Dict[str, Any], ssl.SSLContext], + server_side: Optional[bool] = None, ) -> ssl.SSLContext: """Try to convert an ``ssl_options`` dictionary to an `~ssl.SSLContext` object. @@ -570,19 +571,34 @@ def ssl_options_to_context( `~ssl.SSLContext` equivalent, and may be used when a component which accepts both forms needs to upgrade to the `~ssl.SSLContext` version to use features like SNI or NPN. + + .. versionchanged:: 6.2 + + Added server_side argument. Omitting this argument will + result in a DeprecationWarning on Python 3.10. + """ if isinstance(ssl_options, ssl.SSLContext): return ssl_options assert isinstance(ssl_options, dict) assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options - # Can't use create_default_context since this interface doesn't - # tell us client vs server. - context = ssl.SSLContext(ssl_options.get("ssl_version", ssl.PROTOCOL_SSLv23)) + # TODO: Now that we have the server_side argument, can we switch to + # create_default_context or would that change behavior? + default_version = ssl.PROTOCOL_TLS + if server_side: + default_version = ssl.PROTOCOL_TLS_SERVER + elif server_side is not None: + default_version = ssl.PROTOCOL_TLS_CLIENT + context = ssl.SSLContext(ssl_options.get("ssl_version", default_version)) if "certfile" in ssl_options: context.load_cert_chain( ssl_options["certfile"], ssl_options.get("keyfile", None) ) if "cert_reqs" in ssl_options: + if ssl_options["cert_reqs"] == ssl.CERT_NONE: + # This may have been set automatically by PROTOCOL_TLS_CLIENT but is + # incompatible with CERT_NONE so we must manually clear it. + context.check_hostname = False context.verify_mode = ssl_options["cert_reqs"] if "ca_certs" in ssl_options: context.load_verify_locations(ssl_options["ca_certs"]) @@ -601,6 +617,7 @@ def ssl_wrap_socket( socket: socket.socket, ssl_options: Union[Dict[str, Any], ssl.SSLContext], server_hostname: Optional[str] = None, + server_side: Optional[bool] = None, **kwargs: Any ) -> ssl.SSLSocket: """Returns an ``ssl.SSLSocket`` wrapping the given socket. @@ -610,14 +627,23 @@ def ssl_wrap_socket( keyword arguments are passed to ``wrap_socket`` (either the `~ssl.SSLContext` method or the `ssl` module function as appropriate). + + .. versionchanged:: 6.2 + + Added server_side argument. Omitting this argument will + result in a DeprecationWarning on Python 3.10. """ - context = ssl_options_to_context(ssl_options) + context = ssl_options_to_context(ssl_options, server_side=server_side) + if server_side is None: + server_side = False if ssl.HAS_SNI: # In python 3.4, wrap_socket only accepts the server_hostname # argument if HAS_SNI is true. # TODO: add a unittest (python added server-side SNI support in 3.4) # In the meantime it can be manually tested with # python3 -m tornado.httpclient https://sni.velox.ch - return context.wrap_socket(socket, server_hostname=server_hostname, **kwargs) + return context.wrap_socket( + socket, server_hostname=server_hostname, server_side=server_side, **kwargs + ) else: - return context.wrap_socket(socket, **kwargs) + return context.wrap_socket(socket, server_side=server_side, **kwargs) diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py index 5e9c776d02..a244777a37 100644 --- a/tornado/platform/asyncio.py +++ b/tornado/platform/asyncio.py @@ -32,6 +32,7 @@ import sys import threading import typing +import warnings from tornado.gen import convert_yielded from tornado.ioloop import IOLoop, _Selectable @@ -190,7 +191,9 @@ def _handle_events(self, fd: int, events: int) -> None: def start(self) -> None: try: - old_loop = asyncio.get_event_loop() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + old_loop = asyncio.get_event_loop() except (RuntimeError, AssertionError): old_loop = None # type: ignore try: @@ -321,10 +324,18 @@ def initialize(self, **kwargs: Any) -> None: # type: ignore def close(self, all_fds: bool = False) -> None: if self.is_current: - self.clear_current() + with warnings.catch_warnings(): + # We can't get here unless the warning in make_current + # was swallowed, so swallow the one from clear_current too. + warnings.simplefilter("ignore", DeprecationWarning) + self.clear_current() super().close(all_fds=all_fds) def make_current(self) -> None: + warnings.warn( + "make_current is deprecated; start the event loop first", + DeprecationWarning, + ) if not self.is_current: try: self.old_asyncio = asyncio.get_event_loop() @@ -391,8 +402,25 @@ class AnyThreadEventLoopPolicy(_BasePolicy): # type: ignore .. versionadded:: 5.0 + .. deprecated:: 6.2 + + ``AnyThreadEventLoopPolicy`` affects the implicit creation + of an event loop, which is deprecated in Python 3.10 and + will be removed in a future version of Python. At that time + ``AnyThreadEventLoopPolicy`` will no longer be useful. + If you are relying on it, use `asyncio.new_event_loop` + or `asyncio.run` explicitly in any non-main threads that + need event loops. """ + def __init__(self) -> None: + super().__init__() + warnings.warn( + "AnyThreadEventLoopPolicy is deprecated, use asyncio.run " + "or asyncio.new_event_loop instead", + DeprecationWarning, + ) + def get_event_loop(self) -> asyncio.AbstractEventLoop: try: return super().get_event_loop() diff --git a/tornado/test/asyncio_test.py b/tornado/test/asyncio_test.py index 3f9f3389a2..bdb27e8293 100644 --- a/tornado/test/asyncio_test.py +++ b/tornado/test/asyncio_test.py @@ -12,6 +12,7 @@ import asyncio import unittest +import warnings from concurrent.futures import ThreadPoolExecutor from tornado import gen @@ -26,12 +27,15 @@ class AsyncIOLoopTest(AsyncTestCase): def get_new_ioloop(self): - io_loop = AsyncIOLoop() + io_loop = AsyncIOLoop(make_current=False) return io_loop def test_asyncio_callback(self): # Basic test that the asyncio loop is set up correctly. - asyncio.get_event_loop().call_soon(self.stop) + async def add_callback(): + asyncio.get_event_loop().call_soon(self.stop) + + self.asyncio_loop.run_until_complete(add_callback()) self.wait() @gen_test @@ -89,43 +93,42 @@ async def native_coroutine_with_adapter2(): # Asyncio only supports coroutines that yield asyncio-compatible # Futures (which our Future is since 5.0). self.assertEqual( - asyncio.get_event_loop().run_until_complete( - native_coroutine_without_adapter() - ), + self.asyncio_loop.run_until_complete(native_coroutine_without_adapter()), 42, ) self.assertEqual( - asyncio.get_event_loop().run_until_complete( - native_coroutine_with_adapter() - ), + self.asyncio_loop.run_until_complete(native_coroutine_with_adapter()), 42, ) self.assertEqual( - asyncio.get_event_loop().run_until_complete( - native_coroutine_with_adapter2() - ), + self.asyncio_loop.run_until_complete(native_coroutine_with_adapter2()), 42, ) + # I'm not entirely sure why this manual cleanup is necessary but without + # it we have at-a-distance failures in ioloop_test.TestIOLoopCurrent. + asyncio.set_event_loop(None) class LeakTest(unittest.TestCase): def setUp(self): # Trigger a cleanup of the mapping so we start with a clean slate. - AsyncIOLoop().close() + AsyncIOLoop(make_current=False).close() # If we don't clean up after ourselves other tests may fail on # py34. self.orig_policy = asyncio.get_event_loop_policy() asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) def tearDown(self): - asyncio.get_event_loop().close() + asyncio.get_event_loop_policy().get_event_loop().close() asyncio.set_event_loop_policy(self.orig_policy) def test_ioloop_close_leak(self): orig_count = len(IOLoop._ioloop_for_asyncio) for i in range(10): # Create and close an AsyncIOLoop using Tornado interfaces. - loop = AsyncIOLoop() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loop = AsyncIOLoop() loop.close() new_count = len(IOLoop._ioloop_for_asyncio) - orig_count self.assertEqual(new_count, 0) @@ -171,17 +174,21 @@ def get_and_close_event_loop(): return future.result() def run_policy_test(self, accessor, expected_type): - # With the default policy, non-main threads don't get an event - # loop. - self.assertRaises( - (RuntimeError, AssertionError), self.executor.submit(accessor).result - ) - # Set the policy and we can get a loop. - asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) - self.assertIsInstance(self.executor.submit(accessor).result(), expected_type) - # Clean up to silence leak warnings. Always use asyncio since - # IOLoop doesn't (currently) close the underlying loop. - self.executor.submit(lambda: asyncio.get_event_loop().close()).result() # type: ignore + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + # With the default policy, non-main threads don't get an event + # loop. + self.assertRaises( + (RuntimeError, AssertionError), self.executor.submit(accessor).result + ) + # Set the policy and we can get a loop. + asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) + self.assertIsInstance( + self.executor.submit(accessor).result(), expected_type + ) + # Clean up to silence leak warnings. Always use asyncio since + # IOLoop doesn't (currently) close the underlying loop. + self.executor.submit(lambda: asyncio.get_event_loop().close()).result() # type: ignore def test_asyncio_accessor(self): self.run_policy_test(asyncio.get_event_loop, asyncio.AbstractEventLoop) diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py index 6021d6759b..6b6aced057 100644 --- a/tornado/test/httpclient_test.py +++ b/tornado/test/httpclient_test.py @@ -754,7 +754,7 @@ def test_str(self): class SyncHTTPClientTest(unittest.TestCase): def setUp(self): - self.server_ioloop = IOLoop() + self.server_ioloop = IOLoop(make_current=False) event = threading.Event() @gen.coroutine diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 58696e238e..146ed781ca 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -181,7 +181,9 @@ def get_ssl_version(self): class SSLContextTest(BaseSSLTest, SSLTestMixin): def get_ssl_options(self): - context = ssl_options_to_context(AsyncHTTPSTestCase.get_ssl_options(self)) + context = ssl_options_to_context( + AsyncHTTPSTestCase.get_ssl_options(self), server_side=True + ) assert isinstance(context, ssl.SSLContext) return context diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py index 6fd41540f3..c28516506b 100644 --- a/tornado/test/ioloop_test.py +++ b/tornado/test/ioloop_test.py @@ -16,8 +16,18 @@ from tornado import gen from tornado.ioloop import IOLoop, TimeoutError, PeriodicCallback from tornado.log import app_log -from tornado.testing import AsyncTestCase, bind_unused_port, ExpectLog, gen_test -from tornado.test.util import skipIfNonUnix, skipOnTravis +from tornado.testing import ( + AsyncTestCase, + bind_unused_port, + ExpectLog, + gen_test, + setup_with_context_manager, +) +from tornado.test.util import ( + ignore_deprecation, + skipIfNonUnix, + skipOnTravis, +) import typing @@ -410,7 +420,7 @@ def test_init_close_race(self): # threads. def f(): for i in range(10): - loop = IOLoop() + loop = IOLoop(make_current=False) loop.close() yield gen.multi([self.io_loop.run_in_executor(None, f) for i in range(2)]) @@ -420,6 +430,7 @@ def f(): # automatically set as current. class TestIOLoopCurrent(unittest.TestCase): def setUp(self): + setup_with_context_manager(self, ignore_deprecation()) self.io_loop = None # type: typing.Optional[IOLoop] IOLoop.clear_current() @@ -466,6 +477,10 @@ def test_force_current(self): class TestIOLoopCurrentAsync(AsyncTestCase): + def setUp(self): + super().setUp() + setup_with_context_manager(self, ignore_deprecation()) + @gen_test def test_clear_without_current(self): # If there is no current IOLoop, clear_current is a no-op (but @@ -557,7 +572,7 @@ def sync_func(): class TestIOLoopRunSync(unittest.TestCase): def setUp(self): - self.io_loop = IOLoop() + self.io_loop = IOLoop(make_current=False) def tearDown(self): self.io_loop.close() diff --git a/tornado/test/iostream_test.py b/tornado/test/iostream_test.py index 8e8e9f477a..b098c9aacd 100644 --- a/tornado/test/iostream_test.py +++ b/tornado/test/iostream_test.py @@ -12,7 +12,7 @@ from tornado.httputil import HTTPHeaders from tornado.locks import Condition, Event from tornado.log import gen_log -from tornado.netutil import ssl_wrap_socket +from tornado.netutil import ssl_options_to_context, ssl_wrap_socket from tornado.platform.asyncio import AddThreadSelectorEventLoop from tornado.tcpserver import TCPServer from tornado.testing import ( @@ -23,7 +23,12 @@ ExpectLog, gen_test, ) -from tornado.test.util import skipIfNonUnix, refusing_port, skipPypy3V58 +from tornado.test.util import ( + skipIfNonUnix, + refusing_port, + skipPypy3V58, + ignore_deprecation, +) from tornado.web import RequestHandler, Application import asyncio import errno @@ -900,11 +905,11 @@ def _make_client_iostream(self, connection, **kwargs): class TestIOStreamSSL(TestIOStreamMixin, AsyncTestCase): def _make_server_iostream(self, connection, **kwargs): - connection = ssl.wrap_socket( + ssl_ctx = ssl_options_to_context(_server_ssl_options(), server_side=True) + connection = ssl_ctx.wrap_socket( connection, server_side=True, do_handshake_on_connect=False, - **_server_ssl_options() ) return SSLIOStream(connection, **kwargs) @@ -919,7 +924,7 @@ def _make_client_iostream(self, connection, **kwargs): # instead of an ssl_options dict to the SSLIOStream constructor. class TestIOStreamSSLContext(TestIOStreamMixin, AsyncTestCase): def _make_server_iostream(self, connection, **kwargs): - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) context.load_cert_chain( os.path.join(os.path.dirname(__file__), "test.crt"), os.path.join(os.path.dirname(__file__), "test.key"), @@ -930,7 +935,9 @@ def _make_server_iostream(self, connection, **kwargs): return SSLIOStream(connection, **kwargs) def _make_client_iostream(self, connection, **kwargs): - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE return SSLIOStream(connection, ssl_options=context, **kwargs) @@ -1076,8 +1083,11 @@ def connect_to_server(self, server_cls): # to openssl 1.1.c. Other platforms might be affected with # newer openssl too). Disable it until we figure out # what's up. - ssl_ctx.options |= getattr(ssl, "OP_NO_TLSv1_3", 0) - client = SSLIOStream(socket.socket(), ssl_options=ssl_ctx) + # Update 2021-12-28: Still happening with Python 3.10 on + # Windows. OP_NO_TLSv1_3 now raises a DeprecationWarning. + with ignore_deprecation(): + ssl_ctx.options |= getattr(ssl, "OP_NO_TLSv1_3", 0) + client = SSLIOStream(socket.socket(), ssl_options=ssl_ctx) yield client.connect(("127.0.0.1", port)) self.assertIsNotNone(client.socket.cipher()) finally: diff --git a/tornado/test/process_test.py b/tornado/test/process_test.py index 6ff8efde7b..ab290085b3 100644 --- a/tornado/test/process_test.py +++ b/tornado/test/process_test.py @@ -76,15 +76,15 @@ def get_url(path): sock.close() return try: - if asyncio is not None: - # Reset the global asyncio event loop, which was put into - # a broken state by the fork. - asyncio.set_event_loop(asyncio.new_event_loop()) if id in (0, 1): self.assertEqual(id, task_id()) - server = HTTPServer(self.get_app()) - server.add_sockets([sock]) - IOLoop.current().start() + + async def f(): + server = HTTPServer(self.get_app()) + server.add_sockets([sock]) + await asyncio.Event().wait() + + asyncio.run(f()) elif id == 2: self.assertEqual(id, task_id()) sock.close() diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index eadd4ed303..62bd4830c8 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -537,11 +537,14 @@ def create_client(self, **kwargs): ) def test_ssl_options(self): - resp = self.fetch("/hello", ssl_options={}) + resp = self.fetch("/hello", ssl_options={"cert_reqs": ssl.CERT_NONE}) self.assertEqual(resp.body, b"Hello world!") def test_ssl_context(self): - resp = self.fetch("/hello", ssl_options=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) + ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + resp = self.fetch("/hello", ssl_options=ssl_ctx) self.assertEqual(resp.body, b"Hello world!") def test_ssl_options_handshake_fail(self): @@ -555,8 +558,8 @@ def test_ssl_options_handshake_fail(self): def test_ssl_context_handshake_fail(self): with ExpectLog(gen_log, "SSL Error|Uncaught exception"): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED + # CERT_REQUIRED is set by default. + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) with self.assertRaises(ssl.SSLError): self.fetch("/hello", ssl_options=ctx, raise_error=True) diff --git a/tornado/test/testing_test.py b/tornado/test/testing_test.py index 37cb24602a..a307c4bbf0 100644 --- a/tornado/test/testing_test.py +++ b/tornado/test/testing_test.py @@ -1,6 +1,7 @@ from tornado import gen, ioloop from tornado.httpserver import HTTPServer from tornado.locks import Event +from tornado.test.util import ignore_deprecation from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, bind_unused_port, gen_test from tornado.web import Application import asyncio @@ -333,7 +334,11 @@ def get_new_ioloop(self): def setUp(self): # This simulates the effect of an asyncio test harness like # pytest-asyncio. - self.orig_loop = asyncio.get_event_loop() + with ignore_deprecation(): + try: + self.orig_loop = asyncio.get_event_loop() + except RuntimeError: + self.orig_loop = None self.new_loop = asyncio.new_event_loop() asyncio.set_event_loop(self.new_loop) super().setUp() diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py index 661953d73e..7f983a73e7 100644 --- a/tornado/test/twisted_test.py +++ b/tornado/test/twisted_test.py @@ -13,31 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -import asyncio -import logging -import signal import unittest -import warnings -from tornado.escape import utf8 -from tornado import gen -from tornado.httpclient import AsyncHTTPClient -from tornado.httpserver import HTTPServer -from tornado.ioloop import IOLoop -from tornado.testing import bind_unused_port, AsyncTestCase, gen_test -from tornado.web import RequestHandler, Application +from tornado.testing import AsyncTestCase, gen_test try: from twisted.internet.defer import ( # type: ignore - Deferred, inlineCallbacks, returnValue, ) - from twisted.internet.protocol import Protocol # type: ignore - from twisted.internet.asyncioreactor import AsyncioSelectorReactor # type: ignore - from twisted.web.client import Agent, readBody # type: ignore - from twisted.web.resource import Resource # type: ignore - from twisted.web.server import Site # type: ignore have_twisted = True except ImportError: @@ -49,173 +33,6 @@ skipIfNoTwisted = unittest.skipUnless(have_twisted, "twisted module not present") -def save_signal_handlers(): - saved = {} - signals = [signal.SIGINT, signal.SIGTERM] - if hasattr(signal, "SIGCHLD"): - signals.append(signal.SIGCHLD) - for sig in signals: - saved[sig] = signal.getsignal(sig) - if "twisted" in repr(saved): - # This indicates we're not cleaning up after ourselves properly. - raise Exception("twisted signal handlers already installed") - return saved - - -def restore_signal_handlers(saved): - for sig, handler in saved.items(): - signal.signal(sig, handler) - - -# Test various combinations of twisted and tornado http servers, -# http clients, and event loop interfaces. - - -@skipIfNoTwisted -class CompatibilityTests(unittest.TestCase): - def setUp(self): - self.saved_signals = save_signal_handlers() - self.saved_policy = asyncio.get_event_loop_policy() - if hasattr(asyncio, "WindowsSelectorEventLoopPolicy"): - # Twisted requires a selector event loop, even if Tornado is - # doing its own tricks in AsyncIOLoop to support proactors. - # Setting an AddThreadSelectorEventLoop exposes various edge - # cases so just use a regular selector. - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # type: ignore - self.io_loop = IOLoop() - self.io_loop.make_current() - self.reactor = AsyncioSelectorReactor() - - def tearDown(self): - self.reactor.disconnectAll() - self.io_loop.clear_current() - self.io_loop.close(all_fds=True) - asyncio.set_event_loop_policy(self.saved_policy) - restore_signal_handlers(self.saved_signals) - - def start_twisted_server(self): - class HelloResource(Resource): - isLeaf = True - - def render_GET(self, request): - return b"Hello from twisted!" - - site = Site(HelloResource()) - port = self.reactor.listenTCP(0, site, interface="127.0.0.1") - self.twisted_port = port.getHost().port - - def start_tornado_server(self): - class HelloHandler(RequestHandler): - def get(self): - self.write("Hello from tornado!") - - app = Application([("/", HelloHandler)], log_function=lambda x: None) - server = HTTPServer(app) - sock, self.tornado_port = bind_unused_port() - server.add_sockets([sock]) - - def run_reactor(self): - # In theory, we can run the event loop through Tornado, - # Twisted, or asyncio interfaces. However, since we're trying - # to avoid installing anything as the global event loop, only - # the twisted interface gets everything wired up correectly - # without extra hacks. This method is a part of a - # no-longer-used generalization that allowed us to test - # different combinations. - self.stop_loop = self.reactor.stop - self.stop = self.reactor.stop - self.reactor.run() - - def tornado_fetch(self, url, runner): - client = AsyncHTTPClient() - fut = asyncio.ensure_future(client.fetch(url)) - fut.add_done_callback(lambda f: self.stop_loop()) - runner() - return fut.result() - - def twisted_fetch(self, url, runner): - # http://twistedmatrix.com/documents/current/web/howto/client.html - chunks = [] - client = Agent(self.reactor) - d = client.request(b"GET", utf8(url)) - - class Accumulator(Protocol): - def __init__(self, finished): - self.finished = finished - - def dataReceived(self, data): - chunks.append(data) - - def connectionLost(self, reason): - self.finished.callback(None) - - def callback(response): - finished = Deferred() - response.deliverBody(Accumulator(finished)) - return finished - - d.addCallback(callback) - - def shutdown(failure): - if hasattr(self, "stop_loop"): - self.stop_loop() - elif failure is not None: - # loop hasn't been initialized yet; try our best to - # get an error message out. (the runner() interaction - # should probably be refactored). - try: - failure.raiseException() - except: - logging.error("exception before starting loop", exc_info=True) - - d.addBoth(shutdown) - runner() - self.assertTrue(chunks) - return b"".join(chunks) - - def twisted_coroutine_fetch(self, url, runner): - body = [None] - - @gen.coroutine - def f(): - # This is simpler than the non-coroutine version, but it cheats - # by reading the body in one blob instead of streaming it with - # a Protocol. - client = Agent(self.reactor) - response = yield client.request(b"GET", utf8(url)) - with warnings.catch_warnings(): - # readBody has a buggy DeprecationWarning in Twisted 15.0: - # https://twistedmatrix.com/trac/changeset/43379 - warnings.simplefilter("ignore", category=DeprecationWarning) - body[0] = yield readBody(response) - self.stop_loop() - - self.io_loop.add_callback(f) - runner() - return body[0] - - def testTwistedServerTornadoClientReactor(self): - self.start_twisted_server() - response = self.tornado_fetch( - "http://127.0.0.1:%d" % self.twisted_port, self.run_reactor - ) - self.assertEqual(response.body, b"Hello from twisted!") - - def testTornadoServerTwistedClientReactor(self): - self.start_tornado_server() - response = self.twisted_fetch( - "http://127.0.0.1:%d" % self.tornado_port, self.run_reactor - ) - self.assertEqual(response, b"Hello from tornado!") - - def testTornadoServerTwistedCoroutineClientReactor(self): - self.start_tornado_server() - response = self.twisted_coroutine_fetch( - "http://127.0.0.1:%d" % self.tornado_port, self.run_reactor - ) - self.assertEqual(response, b"Hello from tornado!") - - @skipIfNoTwisted class ConvertDeferredTest(AsyncTestCase): @gen_test diff --git a/tornado/testing.py b/tornado/testing.py index 2e08884192..088530cb76 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -20,6 +20,7 @@ import socket import sys import unittest +import warnings from tornado import gen from tornado.httpclient import AsyncHTTPClient, HTTPResponse @@ -160,6 +161,17 @@ def test_http_fetch(self): response = self.wait() # Test contents of response self.assertIn("FriendFeed", response.body) + + .. deprecated:: 6.2 + + AsyncTestCase and AsyncHTTPTestCase are deprecated due to changes + in future versions of Python (after 3.10). The interfaces used + in this class are incompatible with the deprecation and intended + removal of certain methods related to the idea of a "current" + event loop while no event loop is actually running. Use + `unittest.IsolatedAsyncioTestCase` instead. Note that this class + does not emit DeprecationWarnings until better migration guidance + can be provided. """ def __init__(self, methodName: str = "runTest") -> None: @@ -180,9 +192,49 @@ def __init__(self, methodName: str = "runTest") -> None: self._test_generator = None # type: Optional[Union[Generator, Coroutine]] def setUp(self) -> None: + setup_with_context_manager(self, warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="There is no current event loop", + category=DeprecationWarning, + module=r"tornado\..*", + ) super().setUp() - self.io_loop = self.get_new_ioloop() - self.io_loop.make_current() + # NOTE: this code attempts to navigate deprecation warnings introduced + # in Python 3.10. The idea of an implicit current event loop is + # deprecated in that version, with the intention that tests like this + # explicitly create a new event loop and run on it. However, other + # packages such as pytest-asyncio (as of version 0.16.0) still rely on + # the implicit current event loop and we want to be compatible with them + # (even when run on 3.10, but not, of course, on the future version of + # python that removes the get/set_event_loop methods completely). + # + # Deprecation warnings were introduced inconsistently: + # asyncio.get_event_loop warns, but + # asyncio.get_event_loop_policy().get_event_loop does not. Similarly, + # none of the set_event_loop methods warn, although comments on + # https://bugs.python.org/issue39529 indicate that they are also + # intended for future removal. + # + # Therefore, we first attempt to access the event loop with the + # (non-warning) policy method, and if it fails, fall back to creating a + # new event loop. We do not have effective test coverage of the + # new event loop case; this will have to be watched when/if + # get_event_loop is actually removed. + self.should_close_asyncio_loop = False + try: + self.asyncio_loop = asyncio.get_event_loop_policy().get_event_loop() + except Exception: + self.asyncio_loop = asyncio.new_event_loop() + self.should_close_asyncio_loop = True + + async def get_loop() -> IOLoop: + return self.get_new_ioloop() + + self.io_loop = self.asyncio_loop.run_until_complete(get_loop()) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.io_loop.make_current() def tearDown(self) -> None: # Native coroutines tend to produce warnings if they're not @@ -196,7 +248,7 @@ def tearDown(self) -> None: tasks = asyncio.Task.all_tasks(asyncio_loop) # Tasks that are done may still appear here and may contain # non-cancellation exceptions, so filter them out. - tasks = [t for t in tasks if not t.done()] + tasks = [t for t in tasks if not t.done()] # type: ignore for t in tasks: t.cancel() # Allow the tasks to run and finalize themselves (which means @@ -217,13 +269,17 @@ def tearDown(self) -> None: # Clean up Subprocess, so it can be used again with a new ioloop. Subprocess.uninitialize() - self.io_loop.clear_current() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.io_loop.clear_current() if not isinstance(self.io_loop, _NON_OWNED_IOLOOPS): # Try to clean up any file descriptors left open in the ioloop. # This avoids leaks, especially when tests are run repeatedly # in the same process with autoreload (because curl does not # set FD_CLOEXEC on its file descriptors) self.io_loop.close(all_fds=True) + if self.should_close_asyncio_loop: + self.asyncio_loop.close() super().tearDown() # In case an exception escaped or the StackContext caught an exception # when there wasn't a wait() to re-raise it, do so here. @@ -242,7 +298,7 @@ def get_new_ioloop(self) -> IOLoop: loop is being provided by another system (such as ``pytest-asyncio``). """ - return IOLoop() + return IOLoop(make_current=False) def _handle_exception( self, typ: Type[Exception], value: Exception, tb: TracebackType @@ -724,6 +780,14 @@ def __exit__( raise Exception("did not get expected log message") +# From https://nedbatchelder.com/blog/201508/using_context_managers_in_test_setup.html +def setup_with_context_manager(testcase: unittest.TestCase, cm: Any) -> Any: + """Use a contextmanager to setUp a test case.""" + val = cm.__enter__() + testcase.addCleanup(cm.__exit__, None, None, None) + return val + + def main(**kwargs: Any) -> None: """A simple test runner. diff --git a/tox.ini b/tox.ini index 6c267598d3..56e01a810e 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ [tox] envlist = # Basic configurations: Run the tests for each python version. - py36-full,py37-full,py38-full,py39-full,pypy3-full + py37-full,py38-full,py39-full,py310-full,pypy3-full # Build and test the docs with sphinx. docs @@ -27,10 +27,10 @@ whitelist_externals = /bin/sh [testenv] basepython = py3: python3 - py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 pypy3: pypy3 # In theory, it doesn't matter which python version is used here. # In practice, things like changes to the ast module can alter @@ -49,7 +49,7 @@ deps = setenv = # Treat the extension as mandatory in testing (but not on pypy) - {py3,py36,py37,py38,py39}: TORNADO_EXTENSION=1 + {py3,py37,py38,py39,py310}: TORNADO_EXTENSION=1 # CI workers are often overloaded and can cause our tests to exceed # the default timeout of 5s. ASYNC_TEST_TIMEOUT=25 @@ -61,7 +61,7 @@ setenv = # during sdist installation (and it doesn't seem to be # possible to set environment variables during that phase of # tox). - {py3,py36,py37,py38,py39,pypy3}: PYTHONWARNINGS=error:::tornado + {py3,py37,py38,py39,py310,pypy3}: PYTHONWARNINGS=error:::tornado # All non-comment lines but the last must end in a backslash.