From 2498b13d24eaf5121340d5e1791f9ea73be4d672 Mon Sep 17 00:00:00 2001 From: Dmitry Inyutin Date: Thu, 11 Aug 2022 16:46:46 +0300 Subject: [PATCH] Change params between tries --- README.md | 53 ++++++++++- aiohttp_retry/client.py | 191 +++++++++++++++++++++++++++------------- tests/app.py | 8 ++ tests/test_client.py | 73 ++++++++++++++- 4 files changed, 259 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 219bdbf..fa1973f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,31 @@ async def main(): print(response.status) ``` +You can change parameters between attempts by passing multiple requests params: +```python +from aiohttp_retry import RetryClient, RequestParams, ExponentialRetry + +async def main(): + retry_client = RetryClient(raise_for_status=False) + + async with retry_client.requests( + params_list=[ + RequestParams( + method='GET', + path='https://ya.ru', + ), + RequestParams( + method='GET', + path='https://ya.ru', + headers={'some_header': 'some_value'}, + ), + ] + ) as response: + print(response.status) + + await retry_client.close() +``` + You can also add some logic, F.E. logging, on failures by using trace mechanic. ```python import logging @@ -106,6 +131,7 @@ Look tests for more examples. \ ### Documentation `RetryClient` takes the same arguments as ClientSession[[docs](https://docs.aiohttp.org/en/stable/client_reference.html)] \ `RetryClient` has methods: +- request - get - options - head @@ -158,8 +184,31 @@ It can be useful, if server API sometimes response with malformed data. `RetryClient` add *current attempt number* to `request_trace_ctx` (see examples, for more info see [aiohttp doc](https://docs.aiohttp.org/en/stable/client_advanced.html#aiohttp-client-tracing)). -### Change URL between retries -You can change URL between retries by specifying ```url``` as list of urls. Example: +### Change parameters between retries +`RetryClient` also has a method called `requests`. This method should be used if you want to make requests with different params. +```python +@dataclass +class RequestParams: + method: str + path: _RAW_URL_TYPE + trace_request_ctx: Optional[Dict[str, Any]] = None + kwargs: Optional[Dict[str, Any]] = None +``` + +```python +def requests( + self, + params_list: List[RequestParams], + retry_options: Optional[RetryOptionsBase] = None, + raise_for_status: Optional[bool] = None, +) -> _RequestContext: +``` + +You can find an example of usage above or in tests. +But basically `RequestParams` is a structure to define params for `ClientSession.request` func. +`method`, `path`, `headers` `trace_request_ctx` defined outside kwargs, because they are popular. + +There is also an old way to change URL between retries by specifying ```url``` as list of urls. Example: ```python from aiohttp_retry import RetryClient diff --git a/aiohttp_retry/client.py b/aiohttp_retry/client.py index 3be9e98..38e5577 100644 --- a/aiohttp_retry/client.py +++ b/aiohttp_retry/client.py @@ -2,7 +2,20 @@ import logging import sys from abc import abstractmethod -from typing import Any, Callable, Generator, List, Optional, Tuple, Union +from dataclasses import dataclass +from types import TracebackType +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Generator, + List, + Optional, + Tuple, + Type, + Union, +) from aiohttp import ClientResponse, ClientSession, hdrs from aiohttp.typedefs import StrOrURL @@ -36,25 +49,33 @@ def exception(self, msg: str, *args: Any, **kwargs: Any) -> None: pass _URL_TYPE = Union[_RAW_URL_TYPE, List[_RAW_URL_TYPE], Tuple[_RAW_URL_TYPE, ...]] _LoggerType = Union[_Logger, logging.Logger] +RequestFunc = Callable[..., Awaitable[ClientResponse]] + + +@dataclass +class RequestParams: + method: str + path: _RAW_URL_TYPE + headers: Optional[Dict[str, Any]] = None + trace_request_ctx: Optional[Dict[str, Any]] = None + kwargs: Optional[Dict[str, Any]] = None + class _RequestContext: def __init__( self, - request: Callable[..., Any], # Request operation, like POST or GET - method: str, - urls: Tuple[StrOrURL, ...], + request_func: RequestFunc, + params_list: List[RequestParams], logger: _LoggerType, retry_options: RetryOptionsBase, raise_for_status: bool = False, - **kwargs: Any ) -> None: - self._request = request - self._method = method - self._urls = urls + assert len(params_list) > 0 + + self._request_func = request_func + self._params_list = params_list self._logger = logger self._retry_options = retry_options - self._kwargs = kwargs - self._trace_request_ctx = kwargs.pop('trace_request_ctx', {}) self._raise_for_status = raise_for_status self._response: Optional[ClientResponse] = None @@ -71,14 +92,20 @@ async def _do_request(self) -> ClientResponse: current_attempt += 1 try: - response: ClientResponse = await self._request( - self._method, - self._urls[current_attempt - 1], - **self._kwargs, + try: + params = self._params_list[current_attempt - 1] + except IndexError: + params = self._params_list[-1] + + response: ClientResponse = await self._request_func( + method=params.method, + path=params.path, + headers=params.headers, trace_request_ctx={ 'current_attempt': current_attempt, - **self._trace_request_ctx, + **(params.trace_request_ctx or {}), }, + **(params.kwargs or {}), ) if self._is_status_code_ok(response.status) or current_attempt == self._retry_options.attempts: @@ -121,15 +148,20 @@ def __await__(self) -> Generator[Any, None, ClientResponse]: async def __aenter__(self) -> ClientResponse: return await self._do_request() - async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: if self._response is not None: if not self._response.closed: self._response.close() -def _url_to_urls(url: _URL_TYPE, attempts: int) -> Tuple[StrOrURL, ...]: +def _url_to_urls(url: _URL_TYPE) -> Tuple[StrOrURL, ...]: if isinstance(url, str) or isinstance(url, YARL_URL): - return (url,) * attempts + return (url,) if isinstance(url, list): urls = tuple(url) @@ -141,9 +173,6 @@ def _url_to_urls(url: _URL_TYPE, attempts: int) -> Tuple[StrOrURL, ...]: if len(urls) == 0: raise ValueError("you can pass url by str or list/tuple with attempts count size") - if len(urls) < attempts: - return urls + (urls[-1],) * (attempts - len(url)) - return urls @@ -171,40 +200,22 @@ def __init__( self._retry_options: RetryOptionsBase = retry_options or ExponentialRetry() self._raise_for_status = raise_for_status - def __del__(self) -> None: - if getattr(self, '_closed', None) is None: - # in case object was not initialized (__init__ raised an exception) - return - - if not self._closed: - self._logger.warning("Aiohttp retry client was not closed") + @property + def retry_options(self) -> RetryOptionsBase: + return self._retry_options - def _request( + def requests( self, - method: str, - url: _URL_TYPE, + params_list: List[RequestParams], retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, - **kwargs: Any ) -> _RequestContext: - if retry_options is None: - retry_options = self._retry_options - if raise_for_status is None: - raise_for_status = self._raise_for_status - return _RequestContext( - request=self._client.request, - method=method, - urls=_url_to_urls(url, retry_options.attempts), - logger=self._logger, + return self._make_requests( + params_list=params_list, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs ) - @property - def retry_options(self) -> RetryOptionsBase: - return self._retry_options - def request( self, method: str, @@ -213,12 +224,12 @@ def request( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=method, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def get( @@ -228,12 +239,12 @@ def get( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_GET, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def options( @@ -243,12 +254,12 @@ def options( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_OPTIONS, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def head( @@ -257,12 +268,12 @@ def head( retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_HEAD, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def post( @@ -272,12 +283,12 @@ def post( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_POST, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def put( @@ -287,12 +298,12 @@ def put( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_PUT, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def patch( @@ -302,12 +313,12 @@ def patch( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_PATCH, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) def delete( @@ -317,20 +328,74 @@ def delete( raise_for_status: Optional[bool] = None, **kwargs: Any ) -> _RequestContext: - return self._request( + return self._make_request( method=hdrs.METH_DELETE, url=url, retry_options=retry_options, raise_for_status=raise_for_status, - **kwargs + **kwargs, ) async def close(self) -> None: await self._client.close() self._closed = True + def _make_request( + self, + method: str, + url: _URL_TYPE, + retry_options: Optional[RetryOptionsBase] = None, + raise_for_status: Optional[bool] = None, + **kwargs: Any, + ) -> _RequestContext: + url_list = _url_to_urls(url) + params_list = [RequestParams( + method=method, + path=path, + trace_request_ctx=kwargs.pop('trace_request_ctx', None), + kwargs=kwargs, + ) for path in url_list] + + return self._make_requests( + params_list=params_list, + retry_options=retry_options, + raise_for_status=raise_for_status, + **kwargs, + ) + + def _make_requests( + self, + params_list: List[RequestParams], + retry_options: Optional[RetryOptionsBase] = None, + raise_for_status: Optional[bool] = None, + ) -> _RequestContext: + if retry_options is None: + retry_options = self._retry_options + if raise_for_status is None: + raise_for_status = self._raise_for_status + return _RequestContext( + request_func=self._client.request, + params_list=params_list, + logger=self._logger, + retry_options=retry_options, + raise_for_status=raise_for_status, + ) + async def __aenter__(self) -> 'RetryClient': return self - async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: await self.close() + + def __del__(self) -> None: + if getattr(self, '_closed', None) is None: + # in case object was not initialized (__init__ raised an exception) + return + + if not self._closed: + self._logger.warning("Aiohttp retry client was not closed") diff --git a/tests/app.py b/tests/app.py index 8653d43..8007276 100644 --- a/tests/app.py +++ b/tests/app.py @@ -11,6 +11,7 @@ def __init__(self): app.router.add_get('/not_found_error', self.not_found_error_handler) app.router.add_get('/sometimes_error', self.sometimes_error) app.router.add_get('/sometimes_json', self.sometimes_json) + app.router.add_get('/check_headers', self.check_headers) app.router.add_options('/options_handler', self.ping_handler) app.router.add_head('/head_handler', self.ping_handler) @@ -47,6 +48,13 @@ async def sometimes_json(self, _: web.Request) -> web.Response: return web.Response(text='Ok!', status=200) + async def check_headers(self, request: web.Request) -> web.Response: + self.counter += 1 + if request.headers.get('correct_headers') != 'True': + raise web.HTTPNotAcceptable() + + return web.Response(text='Ok!', status=200) + @property def web_app(self) -> web.Application: return self._web_app diff --git a/tests/test_client.py b/tests/test_client.py index 75b6849..83d54fa 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,7 +12,8 @@ ) from yarl import URL -from aiohttp_retry import ExponentialRetry, ListRetry, RetryClient +from aiohttp_retry import ExponentialRetry, ListRetry, RetryClient, retry_options +from aiohttp_retry.client import RequestParams from aiohttp_retry.retry_options import RetryOptionsBase from tests.app import App @@ -364,3 +365,73 @@ async def evaluate_response(response: ClientResponse) -> bool: assert body == {'status': 'Ok!'} assert test_app.counter == 3 + + +async def test_multiply_paths_by_requests(aiohttp_client): + retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) + async with retry_client.requests( + params_list=[ + RequestParams( + method='GET', + path='/internal_error', + ), + RequestParams( + method='GET', + path='/ping', + ), + ] + ) as response: + text = await response.text() + assert response.status == 200 + assert text == 'Ok!' + + assert test_app.counter == 2 + + await retry_client.close() + + +async def test_multiply_methods_by_requests(aiohttp_client): + retry_options = ExponentialRetry(statuses={405}) # method not allowed + retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) + async with retry_client.requests( + params_list=[ + RequestParams( + method='POST', + path='/ping', + ), + RequestParams( + method='GET', + path='/ping', + ), + ] + ) as response: + text = await response.text() + assert response.status == 200 + assert text == 'Ok!' + + await retry_client.close() + + +async def test_change_headers(aiohttp_client): + retry_options = ExponentialRetry(statuses={406}) + retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) + async with retry_client.requests( + params_list=[ + RequestParams( + method='GET', + path='/check_headers', + ), + RequestParams( + method='GET', + path='/check_headers', + headers={'correct_headers': 'True'}, + ), + ] + ) as response: + text = await response.text() + assert response.status == 200 + assert text == 'Ok!' + + assert test_app.counter == 2 + + await retry_client.close()