From bbaa005d9015f989030cec8f95f2759290a02562 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 18 Oct 2020 20:37:04 +0300 Subject: [PATCH 01/12] Relax multidict (#5077) --- CHANGES/5075.feature | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 CHANGES/5075.feature diff --git a/CHANGES/5075.feature b/CHANGES/5075.feature new file mode 100644 index 00000000000..b5cdbb2f4de --- /dev/null +++ b/CHANGES/5075.feature @@ -0,0 +1 @@ +Multidict > 5 is now supported diff --git a/setup.py b/setup.py index ad87571f274..6b4f2399685 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ install_requires = [ 'attrs>=17.3.0', 'chardet>=2.0,<4.0', - 'multidict>=4.5,<6.0', + 'multidict>=4.5', 'async_timeout>=4.0a2,<5.0', 'yarl>=1.0,<2.0', 'idna-ssl>=1.0; python_version<"3.7"', From 0ee92db355ed6cc38b6c7210830b49e8377d3f91 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 18 Oct 2020 20:56:55 +0300 Subject: [PATCH 02/12] Relax multidict --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b4f2399685..a27c34257cc 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ install_requires = [ 'attrs>=17.3.0', 'chardet>=2.0,<4.0', - 'multidict>=4.5', + 'multidict>=4.5,<7.0', 'async_timeout>=4.0a2,<5.0', 'yarl>=1.0,<2.0', 'idna-ssl>=1.0; python_version<"3.7"', From 6db8b67657b637249cd5a0aba67edfdfb00ea11f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 18 Oct 2020 23:25:14 +0300 Subject: [PATCH 03/12] Better typing (#5078) --- aiohttp/client.py | 4 +-- aiohttp/client_proto.py | 2 +- aiohttp/client_reqrep.py | 2 +- aiohttp/tracing.py | 4 +-- aiohttp/web_app.py | 2 +- aiohttp/web_response.py | 8 ++--- aiohttp/web_routedef.py | 2 +- aiohttp/web_runner.py | 2 +- aiohttp/web_urldispatcher.py | 59 +++++++++++++++++++++++++----------- aiohttp/worker.py | 5 ++- setup.cfg | 2 +- 11 files changed, 60 insertions(+), 32 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index d19855b350f..248df1499f7 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -299,7 +299,7 @@ async def _request( data: Any=None, json: Any=None, cookies: Optional[LooseCookies]=None, - headers: LooseHeaders=None, + headers: Optional[LooseHeaders]=None, skip_auto_headers: Optional[Iterable[str]]=None, auth: Optional[BasicAuth]=None, allow_redirects: bool=True, @@ -1089,7 +1089,7 @@ def request( params: Optional[Mapping[str, str]]=None, data: Any=None, json: Any=None, - headers: LooseHeaders=None, + headers: Optional[LooseHeaders]=None, skip_auto_headers: Optional[Iterable[str]]=None, auth: Optional[BasicAuth]=None, allow_redirects: bool=True, diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index f6d191725ef..1f1a65f88c7 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -140,7 +140,7 @@ def set_parser(self, parser: Any, payload: Any) -> None: data, self._tail = self._tail, b'' self.data_received(data) - def set_response_params(self, *, timer: BaseTimerContext=None, + def set_response_params(self, *, timer: Optional[BaseTimerContext]=None, skip_payload: bool=False, read_until_eof: bool=False, auto_decompress: bool=True, diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index a4aa3492a46..42142bf4718 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -978,7 +978,7 @@ async def text(self, return self._body.decode(encoding, errors=errors) # type: ignore - async def json(self, *, encoding: str=None, + async def json(self, *, encoding: Optional[str]=None, loads: JSONDecoder=DEFAULT_JSON_DECODER, content_type: Optional[str]='application/json') -> Any: """Read and decodes JSON response.""" diff --git a/aiohttp/tracing.py b/aiohttp/tracing.py index d78334dcf4f..2a9b2299202 100644 --- a/aiohttp/tracing.py +++ b/aiohttp/tracing.py @@ -1,5 +1,5 @@ from types import SimpleNamespace -from typing import TYPE_CHECKING, Awaitable, Type, TypeVar +from typing import TYPE_CHECKING, Awaitable, Optional, Type, TypeVar import attr from multidict import CIMultiDict # noqa @@ -92,7 +92,7 @@ def __init__( def trace_config_ctx( self, - trace_request_ctx: SimpleNamespace=None + trace_request_ctx: Optional[SimpleNamespace]=None ) -> SimpleNamespace: # noqa """ Return a new trace_config_ctx instance """ return self._trace_config_ctx_factory( diff --git a/aiohttp/web_app.py b/aiohttp/web_app.py index fe0f343746f..65d359ec3e4 100644 --- a/aiohttp/web_app.py +++ b/aiohttp/web_app.py @@ -77,7 +77,7 @@ class Application(MutableMapping[str, Any]): def __init__(self, *, logger: logging.Logger=web_logger, middlewares: Iterable[_Middleware]=(), - handler_args: Mapping[str, Any]=None, + handler_args: Optional[Mapping[str, Any]]=None, client_max_size: int=1024**2, debug: Any=... # mypy doesn't support ellipsis ) -> None: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 57fc6342720..88b1bd355b7 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -507,7 +507,7 @@ def __init__(self, *, content_type: Optional[str]=None, charset: Optional[str]=None, zlib_executor_size: Optional[int]=None, - zlib_executor: Executor=None) -> None: + zlib_executor: Optional[Executor]=None) -> None: if body is not None and text is not None: raise ValueError("body and text are not allowed together") @@ -716,11 +716,11 @@ async def _do_start_compression(self, coding: ContentCoding) -> None: def json_response(data: Any=sentinel, *, - text: str=None, - body: bytes=None, + text: Optional[str]=None, + body: Optional[bytes]=None, status: int=200, reason: Optional[str]=None, - headers: LooseHeaders=None, + headers: Optional[LooseHeaders]=None, content_type: str='application/json', dumps: JSONEncoder=json.dumps) -> Response: if data is not sentinel: diff --git a/aiohttp/web_routedef.py b/aiohttp/web_routedef.py index e09b3c86763..972e940a80a 100644 --- a/aiohttp/web_routedef.py +++ b/aiohttp/web_routedef.py @@ -85,7 +85,7 @@ def __repr__(self) -> str: def register(self, router: UrlDispatcher) -> List[AbstractRoute]: resource = router.add_static(self.prefix, self.path, **self.kwargs) routes = resource.get_info().get('routes', {}) - return routes.values() + return list(routes.values()) def route(method: str, path: str, handler: _HandlerType, diff --git a/aiohttp/web_runner.py b/aiohttp/web_runner.py index 0fed3b86738..da5918051de 100644 --- a/aiohttp/web_runner.py +++ b/aiohttp/web_runner.py @@ -78,7 +78,7 @@ class TCPSite(BaseSite): __slots__ = ('_host', '_port', '_reuse_address', '_reuse_port') def __init__(self, runner: 'BaseRunner', - host: str=None, port: int=None, *, + host: Optional[str]=None, port: Optional[int]=None, *, shutdown_timeout: float=60.0, ssl_context: Optional[SSLContext]=None, backlog: int=128, reuse_address: Optional[bool]=None, diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 957715d8812..8494e7aa52f 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -20,6 +20,7 @@ List, Mapping, Optional, + Pattern, Set, Sized, Tuple, @@ -28,6 +29,7 @@ cast, ) +from typing_extensions import TypedDict from yarl import URL from . import hdrs @@ -69,6 +71,25 @@ _Resolve = Tuple[Optional[AbstractMatchInfo], Set[str]] +class _InfoDict(TypedDict, total=False): + path: str + + formatter: str + pattern: Pattern[str] + + directory: Path + prefix: str + routes: Mapping[str, 'AbstractRoute'] + + app: 'Application' + + domain: str + + rule: 'AbstractRuleMatching' + + http_exception: HTTPException + + class AbstractResource(Sized, Iterable['AbstractRoute']): def __init__(self, *, name: Optional[str]=None) -> None: @@ -106,7 +127,7 @@ def add_prefix(self, prefix: str) -> None: """ @abc.abstractmethod - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: """Return a dict with additional info useful for introspection""" def freeze(self) -> None: @@ -121,8 +142,8 @@ class AbstractRoute(abc.ABC): def __init__(self, method: str, handler: Union[_WebHandler, Type[AbstractView]], *, - expect_handler: _ExpectHandler=None, - resource: AbstractResource=None) -> None: + expect_handler: Optional[_ExpectHandler]=None, + resource: Optional[AbstractResource]=None) -> None: if expect_handler is None: expect_handler = _default_expect_handler @@ -165,7 +186,7 @@ def resource(self) -> Optional[AbstractResource]: return self._resource @abc.abstractmethod - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: """Return a dict with additional info useful for introspection""" @abc.abstractmethod # pragma: no branch @@ -201,7 +222,7 @@ def expect_handler(self) -> _ExpectHandler: def http_exception(self) -> Optional[HTTPException]: return None - def get_info(self) -> Dict[str, str]: + def get_info(self) -> _InfoDict: # type: ignore return self._route.get_info() @property @@ -361,7 +382,7 @@ def _match(self, path: str) -> Optional[Dict[str, str]]: def raw_match(self, path: str) -> bool: return self._path == path - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'path': self._path} def url_for(self) -> URL: # type: ignore @@ -436,7 +457,7 @@ def _match(self, path: str) -> Optional[Dict[str, str]]: def raw_match(self, path: str) -> bool: return self._formatter == path - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'formatter': self._formatter, 'pattern': self._pattern} @@ -549,7 +570,7 @@ def _get_file_hash(byte_array: bytes) -> str: b64 = base64.urlsafe_b64encode(m.digest()) return b64.decode('ascii') - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'directory': self._directory, 'prefix': self._prefix, 'routes': self._routes} @@ -677,7 +698,7 @@ def url_for(self, *args: str, **kwargs: str) -> URL: raise RuntimeError(".url_for() is not supported " "by sub-application root") - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'app': self._app, 'prefix': self._prefix} @@ -710,7 +731,7 @@ async def match(self, request: Request) -> bool: """Return bool if the request satisfies the criteria""" @abc.abstractmethod # pragma: no branch - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: """Return a dict with additional info useful for introspection""" @property @@ -756,7 +777,7 @@ async def match(self, request: Request) -> bool: def match_domain(self, host: str) -> bool: return host.lower() == self._domain - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'domain': self._domain} @@ -788,7 +809,7 @@ def __init__(self, rule: AbstractRuleMatching, app: 'Application') -> None: def canonical(self) -> str: return self._rule.canonical - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'app': self._app, 'rule': self._rule} @@ -825,14 +846,18 @@ def __repr__(self) -> str: @property def name(self) -> Optional[str]: - return self._resource.name # type: ignore + if self._resource is None: + return None + return self._resource.name def url_for(self, *args: str, **kwargs: str) -> URL: """Construct url for route with additional params.""" - return self._resource.url_for(*args, **kwargs) # type: ignore + assert self._resource is not None + return self._resource.url_for(*args, **kwargs) - def get_info(self) -> Dict[str, Any]: - return self._resource.get_info() # type: ignore + def get_info(self) -> _InfoDict: + assert self._resource is not None + return self._resource.get_info() class SystemRoute(AbstractRoute): @@ -848,7 +873,7 @@ def url_for(self, *args: str, **kwargs: str) -> URL: def name(self) -> Optional[str]: return None - def get_info(self) -> Dict[str, Any]: + def get_info(self) -> _InfoDict: return {'http_exception': self._http_exception} async def _handle(self, request: Request) -> StreamResponse: diff --git a/aiohttp/worker.py b/aiohttp/worker.py index 49a26650193..088e6848f8b 100644 --- a/aiohttp/worker.py +++ b/aiohttp/worker.py @@ -121,7 +121,10 @@ def _wait_next_notify(self) -> 'asyncio.Future[bool]': return waiter - def _notify_waiter_done(self, waiter: 'asyncio.Future[bool]'=None) -> None: + def _notify_waiter_done( + self, + waiter: Optional['asyncio.Future[bool]']=None + ) -> None: if waiter is None: waiter = self._notify_waiter if waiter is not None: diff --git a/setup.cfg b/setup.cfg index e25cfa798ff..e75519c75c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,6 +49,7 @@ xfail_strict = true follow_imports = silent strict_optional = True warn_redundant_casts = True +warn_unused_ignores = True # uncomment next lines # to enable strict mypy mode @@ -56,7 +57,6 @@ warn_redundant_casts = True check_untyped_defs = True disallow_any_generics = True disallow_untyped_defs = True -warn_unused_ignores = True [mypy-pytest] From 92843e65ec6811a8db7432ce480e53e7f2ff5bde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 08:19:49 +0000 Subject: [PATCH 04/12] Bump flake8-pyi from 20.5.0 to 20.10.0 (#5083) Bumps [flake8-pyi](https://github.com/ambv/flake8-pyi) from 20.5.0 to 20.10.0.
Commits
  • b5161b6 prepare release 20.10.0
  • a7f9fae Merge pull request #41 from hauntsaninja/py39
  • 08ad8c1 skip typeshed test for older Python versions
  • 5237738 Revert "[probably shouldn't merge] pin flake8"
  • 9810fcf [probably shouldn't merge] pin flake8
  • d3956e3 don't test typeshed with flake8 < 3.8
  • 30e55d9 Run newer Pythons in CI
  • 258b759 Fix for Python 3.9's AST changes
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=flake8-pyi&package-manager=pip&previous-version=20.5.0&new-version=20.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/configuring-github-dependabot-security-updates) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) --- requirements/lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/lint.txt b/requirements/lint.txt index 80c49130c02..9d6dff300ab 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,5 +1,5 @@ mypy==0.770; implementation_name=="cpython" flake8==3.8.4 -flake8-pyi==20.5.0; python_version >= "3.6" +flake8-pyi==20.10.0; python_version >= "3.6" black==20.8b1; python_version >= "3.6" isort==5.6.4 From 8bae74fb1b26dd373d05bc0ab6ad3f02924655f6 Mon Sep 17 00:00:00 2001 From: Jason Hu Date: Mon, 19 Oct 2020 01:43:44 -0700 Subject: [PATCH 05/12] Add aiohttp-sse-client library to third party usage list (#5084) * Update third_party.rst Add aiohttp-sse-client * Add aiohttp-sse-client library to third party list --- CHANGES/5084.doc | 1 + docs/third_party.rst | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 CHANGES/5084.doc diff --git a/CHANGES/5084.doc b/CHANGES/5084.doc new file mode 100644 index 00000000000..675929274c7 --- /dev/null +++ b/CHANGES/5084.doc @@ -0,0 +1 @@ +Add aiohttp-sse-client library to third party usage list. diff --git a/docs/third_party.rst b/docs/third_party.rst index 198f0ca124e..ff1ae30836e 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -269,3 +269,6 @@ period ask to raise the status. - `aiohttp-tus `_ `tus.io `_ protocol implementation for ``aiohttp.web`` applications. Python 3.6+ required. + +- `aiohttp-sse-client `_ + A Server-Sent Event python client base on aiohttp. Python 3.6+ required. From 46a76113dd5333d0df341631b75f5e143b14eb51 Mon Sep 17 00:00:00 2001 From: ben-dl Date: Mon, 19 Oct 2020 18:22:11 +0200 Subject: [PATCH 06/12] Fix type hint on BaseRunner.addresses (#5086) --- CHANGES/5086.bugfix | 1 + aiohttp/web_runner.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 CHANGES/5086.bugfix diff --git a/CHANGES/5086.bugfix b/CHANGES/5086.bugfix new file mode 100644 index 00000000000..5e45a265326 --- /dev/null +++ b/CHANGES/5086.bugfix @@ -0,0 +1 @@ +Fix type hint on BaseRunner.addresses (from List[str] to List[Any]) diff --git a/aiohttp/web_runner.py b/aiohttp/web_runner.py index da5918051de..27ae200c94a 100644 --- a/aiohttp/web_runner.py +++ b/aiohttp/web_runner.py @@ -209,8 +209,8 @@ def server(self) -> Optional[Server]: return self._server @property - def addresses(self) -> List[str]: - ret = [] # type: List[str] + def addresses(self) -> List[Any]: + ret = [] # type: List[Any] for site in self._sites: server = site._server if server is not None: From 860743c1ee862f727c62cecd0010889eaa14c8cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 17:39:58 +0000 Subject: [PATCH 07/12] Bump sphinxcontrib-spelling from 5.4.0 to 7.0.0 (#5089) Bumps [sphinxcontrib-spelling](https://github.com/sphinx-contrib/spelling) from 5.4.0 to 7.0.0.
Commits
  • 635fbe9 describe bug fix for #96 in release history
  • 8b45030 clean up release preamble formatting
  • ee7ad9c Merge pull request #97 from amureki/issues/96/find_spec_value_error
  • 0c194b1 Handle ValueError raised by importlib.util.find_spec
  • 5425b15 Remove obsolete comment and guard in setup()
  • f543a42 Remove unnecessary UnicodeEncodeError (due to Python 3)
  • 2decd3b Use Python 3 super()
  • f203f59 Remove support for end-of-life Python 3.5
  • 1ff579c Simplify and improve tox configuration
  • 29be696 Capitalize "Python" and "Sphinx" in docs and comments
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sphinxcontrib-spelling&package-manager=pip&previous-version=5.4.0&new-version=7.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/configuring-github-dependabot-security-updates) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) --- requirements/doc-spelling.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt index 6efa4ae4f70..a66dce9840e 100644 --- a/requirements/doc-spelling.txt +++ b/requirements/doc-spelling.txt @@ -1,2 +1,2 @@ -r doc.txt -sphinxcontrib-spelling==5.4.0; platform_system!="Windows" # We only use it in Travis CI +sphinxcontrib-spelling==7.0.0; platform_system!="Windows" # We only use it in Travis CI From 5a8bf3a6bdaf0e595ff006d28375c64681ea03ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:19:09 +0300 Subject: [PATCH 08/12] Bump mypy from 0.770 to 0.790 (#5093) Bumps [mypy](https://github.com/python/mypy) from 0.770 to 0.790. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.770...v0.790) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/lint.txt b/requirements/lint.txt index 9d6dff300ab..acb9a4c6e21 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,4 +1,4 @@ -mypy==0.770; implementation_name=="cpython" +mypy==0.790; implementation_name=="cpython" flake8==3.8.4 flake8-pyi==20.10.0; python_version >= "3.6" black==20.8b1; python_version >= "3.6" From a5010394b51ceb55580961403537d90112cbe6fd Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Oct 2020 18:06:39 +0300 Subject: [PATCH 09/12] Don't ceil small timeouts (#5091) --- CHANGES/4850.feature | 1 + aiohttp/helpers.py | 24 ++++++++++++++++-------- docs/client_quickstart.rst | 17 +++++++++++++++++ docs/spelling_wordlist.txt | 2 ++ requirements/ci-wheel.txt | 4 ++-- tests/test_client_functional.py | 12 ------------ tests/test_client_ws_functional.py | 12 ++---------- tests/test_helpers.py | 25 ++++++++++++++++++++++--- tests/test_web_protocol.py | 12 ++---------- tests/test_web_websocket_functional.py | 14 +++----------- 10 files changed, 67 insertions(+), 56 deletions(-) create mode 100644 CHANGES/4850.feature diff --git a/CHANGES/4850.feature b/CHANGES/4850.feature new file mode 100644 index 00000000000..f01f5682df3 --- /dev/null +++ b/CHANGES/4850.feature @@ -0,0 +1 @@ +Don't ceil timeouts that are smaller than 5 seconds. diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 44c02d4aeda..550885f5169 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -515,10 +515,10 @@ def _weakref_handle(info): # type: ignore getattr(ob, name)() -def weakref_handle(ob, name, timeout, loop, ceil_timeout=True): # type: ignore +def weakref_handle(ob, name, timeout, loop): # type: ignore if timeout is not None and timeout > 0: when = loop.time() + timeout - if ceil_timeout: + if timeout >= 5: when = ceil(when) return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name)) @@ -526,7 +526,9 @@ def weakref_handle(ob, name, timeout, loop, ceil_timeout=True): # type: ignore def call_later(cb, timeout, loop): # type: ignore if timeout is not None and timeout > 0: - when = ceil(loop.time() + timeout) + when = loop.time() + timeout + if timeout > 5: + when = ceil(when) return loop.call_at(when, cb) @@ -548,9 +550,12 @@ def close(self) -> None: self._callbacks.clear() def start(self) -> Optional[asyncio.Handle]: - if self._timeout is not None and self._timeout > 0: - at = ceil(self._loop.time() + self._timeout) - return self._loop.call_at(at, self.__call__) + timeout = self._timeout + if timeout is not None and timeout > 0: + when = self._loop.time() + timeout + if timeout >= 5: + when = ceil(when) + return self._loop.call_at(when, self.__call__) else: return None @@ -626,10 +631,13 @@ def timeout(self) -> None: def ceil_timeout(delay: Optional[float]) -> async_timeout.Timeout: - if delay is not None: + if delay is not None and delay > 0: loop = get_running_loop() now = loop.time() - return async_timeout.timeout_at(ceil(now + delay)) + when = now + delay + if delay > 5: + when = ceil(when) + return async_timeout.timeout_at(when) else: return async_timeout.timeout(None) diff --git a/docs/client_quickstart.rst b/docs/client_quickstart.rst index 2354647ebea..e030ce92d38 100644 --- a/docs/client_quickstart.rst +++ b/docs/client_quickstart.rst @@ -444,3 +444,20 @@ Thus the default timeout is:: aiohttp.ClientTimeout(total=5*60, connect=None, sock_connect=None, sock_read=None) + +.. note:: + + *aiohttp* **ceils** timeout if the value is equal or greater than 5 + seconds. The timeout expires at the next integer second greater than + ``current_time + timeout``. + + The ceiling is done for the sake of optimization, when many concurrent tasks + are scheduled to wake-up at the almost same but different absolute times. It + leads to very many event loop wakeups, which kills performance. + + The optimization shifts absolute wakeup times by scheduling them to exactly + the same time as other neighbors, the loop wakes up once-per-second for + timeout expiration. + + Smaller timeouts are not rounded to help testing; in the real life network + timeouts usually greater than tens of seconds. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index e187f65b559..a03b21dc621 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -312,6 +312,8 @@ utils uvloop vcvarsall waituntil +wakeup +wakeups webapp websocket websocket’s diff --git a/requirements/ci-wheel.txt b/requirements/ci-wheel.txt index aa835d69c1e..f33579936ea 100644 --- a/requirements/ci-wheel.txt +++ b/requirements/ci-wheel.txt @@ -17,8 +17,8 @@ yarl==1.4.2 # required c-ares will not build on windows and has build problems on Macos Python<3.7 aiodns==2.0.0; sys_platform=="linux" or sys_platform=="darwin" and python_version>="3.7" -cryptography==2.9.2; platform_machine!="i686" and python_version<"3.8" # no 32-bit wheels; no python 3.9 wheels yet +cryptography==2.9.2; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet trustme==0.6.0; platform_machine!="i686" # no 32-bit wheels codecov==2.1.10 -uvloop==0.12.1; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.7" # MagicStack/uvloop#14 +uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14 idna-ssl==1.1.0; python_version<"3.7" diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 44b43a510ff..53be3bb75bf 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -29,10 +29,6 @@ def fname(here): return here / 'conftest.py' -def ceil(val): - return val - - async def test_keepalive_two_requests_success( aiohttp_client) -> None: async def handler(request): @@ -567,7 +563,6 @@ async def handler(request): async def test_timeout_on_reading_headers(aiohttp_client, mocker) -> None: - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil async def handler(request): resp = web.StreamResponse() @@ -586,8 +581,6 @@ async def handler(request): async def test_timeout_on_conn_reading_headers(aiohttp_client, mocker) -> None: # tests case where user did not set a connection timeout - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil - async def handler(request): resp = web.StreamResponse() await asyncio.sleep(0.1) @@ -605,7 +598,6 @@ async def handler(request): async def test_timeout_on_session_read_timeout(aiohttp_client, mocker) -> None: - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil async def handler(request): resp = web.StreamResponse() @@ -627,7 +619,6 @@ async def handler(request): async def test_read_timeout_between_chunks(aiohttp_client, mocker) -> None: - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil async def handler(request): resp = aiohttp.web.StreamResponse() @@ -653,7 +644,6 @@ async def handler(request): async def test_read_timeout_on_reading_chunks(aiohttp_client, mocker) -> None: - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil async def handler(request): resp = aiohttp.web.StreamResponse() @@ -679,7 +669,6 @@ async def handler(request): async def test_timeout_on_reading_data(aiohttp_client, mocker) -> None: loop = asyncio.get_event_loop() - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil fut = loop.create_future() async def handler(request): @@ -701,7 +690,6 @@ async def handler(request): async def test_timeout_none(aiohttp_client, mocker) -> None: - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil async def handler(request): resp = web.StreamResponse() diff --git a/tests/test_client_ws_functional.py b/tests/test_client_ws_functional.py index 7f6bf3c0e9c..bdcac0ef07e 100644 --- a/tests/test_client_ws_functional.py +++ b/tests/test_client_ws_functional.py @@ -8,14 +8,6 @@ from aiohttp.client_ws import ClientWSTimeout -@pytest.fixture -def ceil(mocker): - def ceil(val): - return val - - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil - - async def test_send_recv_text(aiohttp_client) -> None: async def handler(request): @@ -589,7 +581,7 @@ async def handler(request): await resp.close() -async def test_heartbeat(aiohttp_client, ceil) -> None: +async def test_heartbeat(aiohttp_client) -> None: ping_received = False async def handler(request): @@ -614,7 +606,7 @@ async def handler(request): assert ping_received -async def test_heartbeat_no_pong(aiohttp_client, ceil) -> None: +async def test_heartbeat_no_pong(aiohttp_client) -> None: ping_received = False async def handler(request): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 57c380108c0..065d5c3ffa8 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -306,6 +306,18 @@ def test_timeout_handle(loop) -> None: assert not handle._callbacks +def test_when_timeout_smaller_second(loop) -> None: + timeout = 0.1 + timer = loop.time() + timeout + + handle = helpers.TimeoutHandle(loop, timeout) + when = handle.start()._when + handle.close() + + assert isinstance(when, float) + assert f"{when:.3f}" == f"{timer:.3f}" + + def test_timeout_handle_cb_exc(loop) -> None: handle = helpers.TimeoutHandle(loop, 10.2) cb = mock.Mock() @@ -341,14 +353,14 @@ def test_timer_context_no_task(loop) -> None: async def test_weakref_handle(loop) -> None: cb = mock.Mock() - helpers.weakref_handle(cb, 'test', 0.01, loop, False) + helpers.weakref_handle(cb, 'test', 0.01, loop) await asyncio.sleep(0.1) assert cb.test.called async def test_weakref_handle_weak(loop) -> None: cb = mock.Mock() - helpers.weakref_handle(cb, 'test', 0.01, loop, False) + helpers.weakref_handle(cb, 'test', 0.01, loop) del cb gc.collect() await asyncio.sleep(0.1) @@ -377,11 +389,18 @@ async def test_ceil_timeout_none(loop) -> None: async def test_ceil_timeout_round(loop) -> None: - async with helpers.ceil_timeout(1.5) as cm: + async with helpers.ceil_timeout(7.5) as cm: frac, integer = modf(cm.deadline) assert frac == 0 +async def test_ceil_timeout_small(loop) -> None: + async with helpers.ceil_timeout(1.1) as cm: + frac, integer = modf(cm.deadline) + # a chance for exact integer with zero fraction is negligible + assert frac != 0 + + # -------------------------------- ContentDisposition ------------------- def test_content_disposition() -> None: diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index efc011d7c0f..84d0f493378 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -97,14 +97,6 @@ def write(chunk): return transport -@pytest.fixture -def ceil(mocker): - def ceil(val): - return val - - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil - - async def test_shutdown(srv, transport) -> None: loop = asyncio.get_event_loop() assert transport is srv.transport @@ -430,7 +422,7 @@ async def handle_request(request): async def test_lingering_timeout( - make_srv, transport, ceil, request_handler + make_srv, transport, request_handler ): async def handle_request(request): @@ -523,7 +515,7 @@ async def test_handle_400(srv, buf, transport) -> None: assert b'400 Bad Request' in buf -async def test_keep_alive(make_srv, transport, ceil) -> None: +async def test_keep_alive(make_srv, transport) -> None: loop = asyncio.get_event_loop() srv = make_srv(keepalive_timeout=0.05) future = loop.create_future() diff --git a/tests/test_web_websocket_functional.py b/tests/test_web_websocket_functional.py index 94280cf45f1..35d46bf6594 100644 --- a/tests/test_web_websocket_functional.py +++ b/tests/test_web_websocket_functional.py @@ -9,14 +9,6 @@ from aiohttp.http import WSMsgType -@pytest.fixture -def ceil(mocker): - def ceil(val): - return val - - mocker.patch('aiohttp.helpers.ceil').side_effect = ceil - - async def test_websocket_can_prepare(loop, aiohttp_client) -> None: async def handler(request): @@ -516,7 +508,7 @@ async def handler(request): await closed -async def aiohttp_client_close_handshake(loop, aiohttp_client, ceil): +async def aiohttp_client_close_handshake(loop, aiohttp_client): closed = loop.create_future() @@ -633,7 +625,7 @@ async def handler(request): assert raised -async def test_heartbeat(loop, aiohttp_client, ceil) -> None: +async def test_heartbeat(loop, aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse(heartbeat=0.05) @@ -654,7 +646,7 @@ async def handler(request): await ws.close() -async def test_heartbeat_no_pong(loop, aiohttp_client, ceil) -> None: +async def test_heartbeat_no_pong(loop, aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse(heartbeat=0.05) From ebeecdb0f1e7ecbdf48597c1c03df9c7dda878d1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Oct 2020 18:09:31 +0300 Subject: [PATCH 10/12] Drop dead code --- aiohttp/tcp_helpers.py | 29 +--------------- tests/test_tcp_helpers.py | 69 +-------------------------------------- 2 files changed, 2 insertions(+), 96 deletions(-) diff --git a/aiohttp/tcp_helpers.py b/aiohttp/tcp_helpers.py index 440c1167321..a93a528b345 100644 --- a/aiohttp/tcp_helpers.py +++ b/aiohttp/tcp_helpers.py @@ -5,15 +5,7 @@ from contextlib import suppress from typing import Optional # noqa -__all__ = ('tcp_keepalive', 'tcp_nodelay', 'tcp_cork') - - -if hasattr(socket, 'TCP_CORK'): # pragma: no cover - CORK = socket.TCP_CORK # type: Optional[int] -elif hasattr(socket, 'TCP_NOPUSH'): # pragma: no cover - CORK = socket.TCP_NOPUSH # type: ignore -else: # pragma: no cover - CORK = None +__all__ = ('tcp_keepalive', 'tcp_nodelay') if hasattr(socket, 'SO_KEEPALIVE'): @@ -42,22 +34,3 @@ def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None: with suppress(OSError): sock.setsockopt( socket.IPPROTO_TCP, socket.TCP_NODELAY, value) - - -def tcp_cork(transport: asyncio.Transport, value: bool) -> None: - sock = transport.get_extra_info('socket') - - if CORK is None: - return - - if sock is None: - return - - if sock.family not in (socket.AF_INET, socket.AF_INET6): - return - - value = bool(value) - - with suppress(OSError): - sock.setsockopt( - socket.IPPROTO_TCP, CORK, value) diff --git a/tests/test_tcp_helpers.py b/tests/test_tcp_helpers.py index dbb8c0cf6c4..9ccd10793f9 100644 --- a/tests/test_tcp_helpers.py +++ b/tests/test_tcp_helpers.py @@ -3,7 +3,7 @@ import pytest -from aiohttp.tcp_helpers import CORK, tcp_cork, tcp_nodelay +from aiohttp.tcp_helpers import tcp_nodelay has_ipv6 = socket.has_ipv6 if has_ipv6: @@ -75,70 +75,3 @@ def test_tcp_nodelay_enable_no_socket() -> None: transport = mock.Mock() transport.get_extra_info.return_value = None tcp_nodelay(transport, True) - - -# cork - - -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_tcp_cork_enable() -> None: - transport = mock.Mock() - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - transport.get_extra_info.return_value = s - tcp_cork(transport, True) - assert s.getsockopt(socket.IPPROTO_TCP, CORK) - - -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_set_cork_enable_and_disable() -> None: - transport = mock.Mock() - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - transport.get_extra_info.return_value = s - tcp_cork(transport, True) - assert s.getsockopt(socket.IPPROTO_TCP, CORK) - tcp_cork(transport, False) - assert not s.getsockopt(socket.IPPROTO_TCP, CORK) - - -@pytest.mark.skipif(not has_ipv6, reason="IPv6 is not available") -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_set_cork_enable_ipv6() -> None: - transport = mock.Mock() - with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: - transport.get_extra_info.return_value = s - tcp_cork(transport, True) - assert s.getsockopt(socket.IPPROTO_TCP, CORK) - - -@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), - reason="requires unix sockets") -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_set_cork_enable_unix() -> None: - transport = mock.Mock() - s = mock.Mock(family=socket.AF_UNIX, type=socket.SOCK_STREAM) - transport.get_extra_info.return_value = s - tcp_cork(transport, True) - assert not s.setsockopt.called - - -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_set_cork_enable_no_socket() -> None: - transport = mock.Mock() - transport.get_extra_info.return_value = None - tcp_cork(transport, True) - - -@pytest.mark.skipif(CORK is None, reason="TCP_CORK or TCP_NOPUSH required") -def test_set_cork_exception() -> None: - transport = mock.Mock() - s = mock.Mock() - s.setsockopt = mock.Mock() - s.family = socket.AF_INET - s.setsockopt.side_effect = OSError - transport.get_extra_info.return_value = s - tcp_cork(transport, True) - s.setsockopt.assert_called_with( - socket.IPPROTO_TCP, - CORK, - True - ) From 967b1b906db330ad4bbb4f0de9e4a8f80a177977 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Oct 2020 18:39:02 +0300 Subject: [PATCH 11/12] Debug github context --- .github/workflows/autosquash.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/autosquash.yml b/.github/workflows/autosquash.yml index 5daa9447cb0..7eb5258e5c2 100644 --- a/.github/workflows/autosquash.yml +++ b/.github/workflows/autosquash.yml @@ -28,6 +28,10 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'aio-libs/aiohttp' }} # not awailable for forks, skip the workflow steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" - id: generate_token uses: tibdex/github-app-token@v1 with: From d3219238447f09162f2c38c63ae136a94f39c594 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 21 Oct 2020 09:14:53 +0300 Subject: [PATCH 12/12] Fix inconsistency between Python and C http request parsers. (#4973) They both now correctly parse URLs containing pct-encoded sequences and work with yarl 1.6.0. Co-authored-by: Sviatoslav Sydorenko Co-authored-by: Andrew Svetlov --- CHANGES/4972.bugfix | 1 + CONTRIBUTORS.txt | 1 + aiohttp/_http_parser.pyx | 2 +- aiohttp/web_urldispatcher.py | 54 ++++++++++++++++++++++++------------ requirements/ci-wheel.txt | 2 +- tests/test_http_parser.py | 50 +++++++++++++++++++++++++++++++++ tests/test_urldispatch.py | 26 ++++++++++++++--- vendor/http-parser | 2 +- 8 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 CHANGES/4972.bugfix diff --git a/CHANGES/4972.bugfix b/CHANGES/4972.bugfix new file mode 100644 index 00000000000..6654f8a645d --- /dev/null +++ b/CHANGES/4972.bugfix @@ -0,0 +1 @@ +Fix inconsistency between Python and C http request parsers in parsing pct-encoded URL. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 93d4ab5b1ed..d712817b89c 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -250,6 +250,7 @@ Sergey Ninua Sergey Skripnick Serhii Charykov Serhii Kostel +Serhiy Storchaka Simon Kennedy Sin-Woo Bang Stanislas Plum diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index eb2157f6bb7..04360b89009 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -868,7 +868,7 @@ cdef _parse_url(char* buf_data, size_t length): return URL_build(scheme=schema, user=user, password=password, host=host, port=port, - path=path, query=query, fragment=fragment) + path=path, query_string=query, fragment=fragment, encoded=True) else: raise InvalidURLError("invalid url {!r}".format(buf_data)) finally: diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 8494e7aa52f..08b989e8022 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -31,6 +31,7 @@ from typing_extensions import TypedDict from yarl import URL +from yarl import __version__ as yarl_version # type: ignore from . import hdrs from .abc import AbstractMatchInfo, AbstractRouter, AbstractView @@ -61,6 +62,8 @@ else: BaseDict = dict +YARL_VERSION = tuple(map(int, yarl_version.split('.')[:2])) + HTTP_METHOD_RE = re.compile(r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$") ROUTE_RE = re.compile(r'(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})') PATH_SEP = re.escape('/') @@ -421,9 +424,9 @@ def __init__(self, path: str, *, name: Optional[str]=None) -> None: if '{' in part or '}' in part: raise ValueError("Invalid path '{}'['{}']".format(path, part)) - path = URL.build(path=part).raw_path - formatter += path - pattern += re.escape(path) + part = _requote_path(part) + formatter += part + pattern += re.escape(part) try: compiled = re.compile(pattern) @@ -451,7 +454,7 @@ def _match(self, path: str) -> Optional[Dict[str, str]]: if match is None: return None else: - return {key: URL.build(path=value, encoded=True).path + return {key: _unquote_path(value) for key, value in match.groupdict().items()} def raw_match(self, path: str) -> bool: @@ -462,9 +465,9 @@ def get_info(self) -> _InfoDict: 'pattern': self._pattern} def url_for(self, **parts: str) -> URL: - url = self._formatter.format_map({k: URL.build(path=v).raw_path + url = self._formatter.format_map({k: _quote_path(v) for k, v in parts.items()}) - return URL.build(path=url) + return URL.build(path=url, encoded=True) def __repr__(self) -> str: name = "'" + self.name + "' " if self.name is not None else "" @@ -478,7 +481,7 @@ def __init__(self, prefix: str, *, name: Optional[str]=None) -> None: assert not prefix or prefix.startswith('/'), prefix assert prefix in ('', '/') or not prefix.endswith('/'), prefix super().__init__(name=name) - self._prefix = URL.build(path=prefix).raw_path + self._prefix = _requote_path(prefix) @property def canonical(self) -> str: @@ -535,17 +538,17 @@ def url_for(self, *, filename: Union[str, Path], # type: ignore append_version = self._append_version if isinstance(filename, Path): filename = str(filename) - while filename.startswith('/'): - filename = filename[1:] - filename = '/' + filename + filename = filename.lstrip('/') + url = URL.build(path=self._prefix, encoded=True) # filename is not encoded - url = URL.build(path=self._prefix + filename) + if YARL_VERSION < (1, 6): + url = url / filename.replace('%', '%25') + else: + url = url / filename if append_version: try: - if filename.startswith('/'): - filename = filename[1:] filepath = self._directory.joinpath(filename).resolve() if not self._follow_symlinks: filepath.relative_to(self._directory) @@ -592,8 +595,7 @@ async def resolve(self, request: Request) -> _Resolve: if method not in allowed_methods: return None, allowed_methods - match_dict = {'filename': URL.build(path=path[len(self._prefix)+1:], - encoded=True).path} + match_dict = {'filename': _unquote_path(path[len(self._prefix)+1:])} return (UrlMappingMatchInfo(match_dict, self._routes[method]), allowed_methods) @@ -1032,8 +1034,7 @@ def add_resource(self, path: str, *, if resource.name == name and resource.raw_match(path): return cast(Resource, resource) if not ('{' in path or '}' in path or ROUTE_RE.search(path)): - url = URL.build(path=path) - resource = PlainResource(url.raw_path, name=name) + resource = PlainResource(_requote_path(path), name=name) self.register_resource(resource) return resource resource = DynamicResource(path, name=name) @@ -1152,3 +1153,22 @@ def add_routes(self, for route_def in routes: registered_routes.extend(route_def.register(self)) return registered_routes + + +def _quote_path(value: str) -> str: + if YARL_VERSION < (1, 6): + value = value.replace('%', '%25') + return URL.build(path=value, encoded=False).raw_path + + +def _unquote_path(value: str) -> str: + return URL.build(path=value, encoded=True).path + + +def _requote_path(value: str) -> str: + # Quote non-ascii characters and other characters which must be quoted, + # but preserve existing %-sequences. + result = _quote_path(value) + if '%' in value: + result = result.replace('%25', '%') + return result diff --git a/requirements/ci-wheel.txt b/requirements/ci-wheel.txt index f33579936ea..9178daea2eb 100644 --- a/requirements/ci-wheel.txt +++ b/requirements/ci-wheel.txt @@ -11,7 +11,7 @@ pytest==6.1.1 pytest-cov==2.10.1 pytest-mock==3.3.1 typing_extensions==3.7.4.3 -yarl==1.4.2 +yarl==1.6.1 # Using PEP 508 env markers to control dependency on runtimes: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index a282d52af43..ac4d5f03c76 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -2,6 +2,7 @@ import asyncio from unittest import mock +from urllib.parse import quote import pytest from multidict import CIMultiDict @@ -787,6 +788,55 @@ def test_url_parse_non_strict_mode(parser) -> None: assert payload.is_eof() +@pytest.mark.parametrize( + ('uri', 'path', 'query', 'fragment'), + [ + ('/path%23frag', '/path#frag', {}, ''), + ('/path%2523frag', '/path%23frag', {}, ''), + ('/path?key=value%23frag', '/path', {'key': 'value#frag'}, ''), + ('/path?key=value%2523frag', '/path', {'key': 'value%23frag'}, ''), + ('/path#frag%20', '/path', {}, 'frag '), + ('/path#frag%2520', '/path', {}, 'frag%20'), + ] +) +def test_parse_uri_percent_encoded(parser, uri, path, query, fragment) -> None: + text = ('GET %s HTTP/1.1\r\n\r\n' % (uri,)).encode() + messages, upgrade, tail = parser.feed_data(text) + msg = messages[0][0] + + assert msg.path == uri + assert msg.url == URL(uri) + assert msg.url.path == path + assert msg.url.query == query + assert msg.url.fragment == fragment + + +def test_parse_uri_utf8(parser) -> None: + text = ('GET /путь?ключ=знач#фраг HTTP/1.1\r\n\r\n').encode() + messages, upgrade, tail = parser.feed_data(text) + msg = messages[0][0] + + assert msg.path == '/путь?ключ=знач#фраг' + assert msg.url.path == '/путь' + assert msg.url.query == {'ключ': 'знач'} + assert msg.url.fragment == 'фраг' + + +def test_parse_uri_utf8_percent_encoded(parser) -> None: + text = ( + 'GET %s HTTP/1.1\r\n\r\n' % + quote('/путь?ключ=знач#фраг', safe='/?=#') + ).encode() + messages, upgrade, tail = parser.feed_data(text) + msg = messages[0][0] + + assert msg.path == quote('/путь?ключ=знач#фраг', safe='/?=#') + assert msg.url == URL('/путь?ключ=знач#фраг') + assert msg.url.path == '/путь' + assert msg.url.query == {'ключ': 'знач'} + assert msg.url.fragment == 'фраг' + + @pytest.mark.skipif('HttpRequestParserC' not in dir(aiohttp.http_parser), reason="C based HTTP parser not available") def test_parse_bad_method_for_c_parser_raises(loop, protocol): diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index a05f1d9c7d8..5aa25c76d83 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -467,6 +467,20 @@ def test_add_static_append_version_not_follow_symlink(router, assert '/st/append_version_symlink/data.unknown_mime_type' == str(url) +def test_add_static_quoting(router) -> None: + resource = router.add_static('/пре %2Fфикс', + pathlib.Path(aiohttp.__file__).parent, + name='static') + assert router['static'] is resource + url = resource.url_for(filename='/1 2/файл%2F.txt') + assert url.path == '/пре /фикс/1 2/файл%2F.txt' + assert str(url) == ( + '/%D0%BF%D1%80%D0%B5%20%2F%D1%84%D0%B8%D0%BA%D1%81' + '/1%202/%D1%84%D0%B0%D0%B9%D0%BB%252F.txt' + ) + assert len(resource) == 2 + + def test_plain_not_match(router) -> None: handler = make_handler() router.add_route('GET', '/get/path', handler, name='name') @@ -629,10 +643,14 @@ def test_route_dynamic_with_regex(router) -> None: def test_route_dynamic_quoting(router) -> None: handler = make_handler() - route = router.add_route('GET', r'/{arg}', handler) - - url = route.url_for(arg='1 2/текст') - assert '/1%202/%D1%82%D0%B5%D0%BA%D1%81%D1%82' == str(url) + route = router.add_route('GET', r'/пре %2Fфикс/{arg}', handler) + + url = route.url_for(arg='1 2/текст%2F') + assert url.path == '/пре /фикс/1 2/текст%2F' + assert str(url) == ( + '/%D0%BF%D1%80%D0%B5%20%2F%D1%84%D0%B8%D0%BA%D1%81' + '/1%202/%D1%82%D0%B5%D0%BA%D1%81%D1%82%252F' + ) async def test_regular_match_info(router) -> None: diff --git a/vendor/http-parser b/vendor/http-parser index 2343fd6b521..77310eeb839 160000 --- a/vendor/http-parser +++ b/vendor/http-parser @@ -1 +1 @@ -Subproject commit 2343fd6b5214b2ded2cdcf76de2bf60903bb90cd +Subproject commit 77310eeb839c4251c07184a5db8885a572a08352