diff --git a/CHANGES.rst b/CHANGES.rst index 96f74b3767a..9373390f1f1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -44,6 +44,9 @@ Changes - Do not unquote `+` in match_info values #1816 +- Use Forwarded, X-Forwarded-Scheme and X-Forwarded-Host for better scheme and + host resolution. #1134 + - Fix sub-application middlewares resolution order #1853 - Fix applications comparison #1866 @@ -52,6 +55,10 @@ Changes - Make test server more reliable #1896 +- Use Forwarded, X-Forwarded-Scheme and X-Forwarded-Host for better scheme and + host resolution. #1134 + + 2.0.7 (2017-04-12) ------------------ diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index bdd4977af01..e045278f88a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -62,6 +62,7 @@ Enrique Saez Erich Healy Eugene Chernyshov Eugene Naydenov +Evert Lammerts Frederik Gladhorn Frederik Peter Aalund Gabriel Tremblay diff --git a/aiohttp/abc.py b/aiohttp/abc.py index dc343259fd3..0d2854e66ae 100644 --- a/aiohttp/abc.py +++ b/aiohttp/abc.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Sized + PY_35 = sys.version_info >= (3, 5) diff --git a/aiohttp/backport_cookies.py b/aiohttp/backport_cookies.py index e523e04ba99..94b03c128de 100644 --- a/aiohttp/backport_cookies.py +++ b/aiohttp/backport_cookies.py @@ -40,6 +40,7 @@ import string # pragma: no cover from http.cookies import CookieError, Morsel # pragma: no cover + __all__ = ["CookieError", "BaseCookie", "SimpleCookie"] # pragma: no cover _nulljoin = ''.join # pragma: no cover diff --git a/aiohttp/client.py b/aiohttp/client.py index 34bfe281090..c721a86ec67 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -15,19 +15,20 @@ from . import connector as connector_mod from . import client_exceptions, client_reqrep, hdrs, http, payload from .client_exceptions import * # noqa -from .client_exceptions import (ClientError, ClientOSError, - ServerTimeoutError, WSServerHandshakeError) +from .client_exceptions import (ClientError, ClientOSError, ServerTimeoutError, + WSServerHandshakeError) from .client_reqrep import * # noqa from .client_reqrep import ClientRequest, ClientResponse from .client_ws import ClientWebSocketResponse from .connector import * # noqa from .connector import TCPConnector from .cookiejar import CookieJar -from .helpers import (PY_35, CeilTimeout, TimeoutHandle, - deprecated_noop, sentinel) +from .helpers import (PY_35, CeilTimeout, TimeoutHandle, deprecated_noop, + sentinel) from .http import WS_KEY, WebSocketReader, WebSocketWriter from .streams import FlowControlDataQueue + __all__ = (client_exceptions.__all__ + # noqa client_reqrep.__all__ + # noqa connector_mod.__all__ + # noqa diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 75fbaad8058..98f6a764551 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -2,6 +2,7 @@ from asyncio import TimeoutError + __all__ = ( 'ClientError', diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 6d26ef9d7e8..2c7860a3679 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -20,6 +20,7 @@ from .log import client_logger from .streams import FlowControlStreamReader + try: import cchardet as chardet except ImportError: # pragma: no cover diff --git a/aiohttp/client_ws.py b/aiohttp/client_ws.py index acf19094988..05ef2272580 100644 --- a/aiohttp/client_ws.py +++ b/aiohttp/client_ws.py @@ -5,8 +5,8 @@ from .client_exceptions import ClientError from .helpers import PY_35, PY_352, Timeout, call_later, create_future -from .http import (WS_CLOSED_MESSAGE, WS_CLOSING_MESSAGE, - WebSocketError, WSMessage, WSMsgType) +from .http import (WS_CLOSED_MESSAGE, WS_CLOSING_MESSAGE, WebSocketError, + WSMessage, WSMsgType) class ClientWebSocketResponse: diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 27c2baede94..968b084716b 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from http.cookies import Morsel from math import ceil + from yarl import URL from .abc import AbstractCookieJar diff --git a/aiohttp/formdata.py b/aiohttp/formdata.py index b3a845cd873..a5effd7df73 100644 --- a/aiohttp/formdata.py +++ b/aiohttp/formdata.py @@ -6,6 +6,7 @@ from . import hdrs, multipart, payload from .helpers import guess_filename + __all__ = ('FormData',) diff --git a/aiohttp/hdrs.py b/aiohttp/hdrs.py index f994319e48d..0cb964136e5 100644 --- a/aiohttp/hdrs.py +++ b/aiohttp/hdrs.py @@ -1,6 +1,7 @@ """HTTP Headers constants.""" from multidict import istr + METH_ANY = '*' METH_CONNECT = 'CONNECT' METH_HEAD = 'HEAD' @@ -50,6 +51,7 @@ ETAG = istr('ETAG') EXPECT = istr('EXPECT') EXPIRES = istr('EXPIRES') +FORWARDED = istr('FORWARDED') FROM = istr('FROM') HOST = istr('HOST') IF_MATCH = istr('IF-MATCH') @@ -89,3 +91,5 @@ WANT_DIGEST = istr('WANT-DIGEST') WARNING = istr('WARNING') WWW_AUTHENTICATE = istr('WWW-AUTHENTICATE') +X_FORWARDED_HOST = istr('X-FORWARDED-HOST') +X_FORWARDED_PROTO = istr('X-FORWARDED-PROTO') diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 68d92984557..e3f0ee47703 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -24,6 +24,7 @@ from . import hdrs from .abc import AbstractCookieJar + try: from asyncio import ensure_future except ImportError: diff --git a/aiohttp/http.py b/aiohttp/http.py index 7ee7e76795a..e5f622e01e1 100644 --- a/aiohttp/http.py +++ b/aiohttp/http.py @@ -13,6 +13,7 @@ from .http_writer import (HttpVersion, HttpVersion10, HttpVersion11, PayloadWriter, StreamWriter) + __all__ = ( 'HttpProcessingError', 'RESPONSES', 'SERVER_SOFTWARE', diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 2cb680be540..971f991a70e 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -16,6 +16,7 @@ from .log import internal_logger from .streams import EMPTY_PAYLOAD, FlowControlStreamReader + __all__ = ( 'HttpParser', 'HttpRequestParser', 'HttpResponseParser', 'RawRequestMessage', 'RawResponseMessage') diff --git a/aiohttp/http_websocket.py b/aiohttp/http_websocket.py index 01ae58d7fa4..9752b38e568 100644 --- a/aiohttp/http_websocket.py +++ b/aiohttp/http_websocket.py @@ -15,6 +15,7 @@ from .http_exceptions import HttpBadRequest, HttpProcessingError from .log import ws_logger + __all__ = ('WS_CLOSED_MESSAGE', 'WS_CLOSING_MESSAGE', 'WS_KEY', 'WebSocketReader', 'WebSocketWriter', 'do_handshake', 'WSMessage', 'WebSocketError', 'WSMsgType', 'WSCloseCode') diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 8e6756054bf..49ce32a30c6 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -11,6 +11,7 @@ from .abc import AbstractPayloadWriter from .helpers import create_future, noop + __all__ = ('PayloadWriter', 'HttpVersion', 'HttpVersion10', 'HttpVersion11', 'StreamWriter') diff --git a/aiohttp/log.py b/aiohttp/log.py index cfda0e5f070..48400473129 100644 --- a/aiohttp/log.py +++ b/aiohttp/log.py @@ -1,5 +1,6 @@ import logging + access_logger = logging.getLogger('aiohttp.access') client_logger = logging.getLogger('aiohttp.client') internal_logger = logging.getLogger('aiohttp.internal') diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 635db1e1295..83201f21134 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -18,6 +18,7 @@ from .payload import (BytesPayload, LookupError, Payload, StringPayload, get_payload, payload_type) + __all__ = ('MultipartReader', 'MultipartWriter', 'BodyPartReader', 'BadContentDispositionHeader', 'BadContentDispositionParam', 'parse_content_disposition', 'content_disposition_filename') diff --git a/aiohttp/payload.py b/aiohttp/payload.py index 7ee7876d412..b6334465ce7 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -12,6 +12,7 @@ parse_mimetype, sentinel) from .streams import DEFAULT_LIMIT, DataQueue, EofStream, StreamReader + __all__ = ('PAYLOAD_REGISTRY', 'get_payload', 'payload_type', 'Payload', 'BytesPayload', 'StringPayload', 'StreamReaderPayload', 'IOBasePayload', 'BytesIOPayload', 'BufferedReaderPayload', diff --git a/aiohttp/payload_streamer.py b/aiohttp/payload_streamer.py index 2813469964b..f285e444c09 100644 --- a/aiohttp/payload_streamer.py +++ b/aiohttp/payload_streamer.py @@ -25,6 +25,7 @@ def file_sender(writer, file_name=None): from .payload import Payload, payload_type + __all__ = ('streamer',) diff --git a/aiohttp/pytest_plugin.py b/aiohttp/pytest_plugin.py index e72d6a4eefa..7fb2b041d63 100644 --- a/aiohttp/pytest_plugin.py +++ b/aiohttp/pytest_plugin.py @@ -9,8 +9,9 @@ from aiohttp.web import Application from .test_utils import unused_port as _unused_port -from .test_utils import (RawTestServer, TestClient, TestServer, - loop_context, setup_test_loop, teardown_test_loop) +from .test_utils import (RawTestServer, TestClient, TestServer, loop_context, + setup_test_loop, teardown_test_loop) + try: import uvloop diff --git a/aiohttp/resolver.py b/aiohttp/resolver.py index 102a79b3731..53d4a2f3669 100644 --- a/aiohttp/resolver.py +++ b/aiohttp/resolver.py @@ -3,6 +3,7 @@ from .abc import AbstractResolver + __all__ = ('ThreadedResolver', 'AsyncResolver', 'DefaultResolver') try: diff --git a/aiohttp/streams.py b/aiohttp/streams.py index f64848a279d..9a0e01136a9 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -5,6 +5,7 @@ from . import helpers from .log import internal_logger + __all__ = ( 'EMPTY_PAYLOAD', 'EofStream', 'StreamReader', 'DataQueue', 'ChunksQueue', 'FlowControlStreamReader', diff --git a/aiohttp/web.py b/aiohttp/web.py index e1f3d115efd..e23bd2d5bd3 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -29,6 +29,7 @@ from .web_urldispatcher import PrefixedSubAppResource from .web_ws import * # noqa + __all__ = (web_protocol.__all__ + web_fileresponse.__all__ + web_request.__all__ + @@ -58,6 +59,10 @@ def __init__(self, *, if loop is not None: warnings.warn("loop argument is deprecated", ResourceWarning) + if secure_proxy_ssl_header is not None: + warnings.warn( + "secure_proxy_ssl_header is deprecated", ResourceWarning) + self._debug = debug self._router = router self._secure_proxy_ssl_header = secure_proxy_ssl_header diff --git a/aiohttp/web_exceptions.py b/aiohttp/web_exceptions.py index 0a10d9d1f9a..f04b19a4db0 100644 --- a/aiohttp/web_exceptions.py +++ b/aiohttp/web_exceptions.py @@ -1,5 +1,6 @@ from .web_response import Response + __all__ = ( 'HTTPException', 'HTTPError', diff --git a/aiohttp/web_fileresponse.py b/aiohttp/web_fileresponse.py index 4bd64a8da54..1848525b4e6 100644 --- a/aiohttp/web_fileresponse.py +++ b/aiohttp/web_fileresponse.py @@ -11,6 +11,7 @@ HTTPRequestRangeNotSatisfiable) from .web_response import StreamResponse + __all__ = ('FileResponse',) diff --git a/aiohttp/web_middlewares.py b/aiohttp/web_middlewares.py index 8676a154ab2..8d17afb30f8 100644 --- a/aiohttp/web_middlewares.py +++ b/aiohttp/web_middlewares.py @@ -4,6 +4,7 @@ from aiohttp.web_exceptions import HTTPMovedPermanently from aiohttp.web_urldispatcher import SystemRoute + __all__ = ( 'normalize_path_middleware', ) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index f7bfdbef6b5..38d30c39342 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -18,6 +18,7 @@ from .web_request import BaseRequest from .web_response import Response + __all__ = ('RequestHandler', 'RequestPayloadError') ERROR = http.RawRequestMessage( diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 204c7b5a258..b497b5a7f08 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -3,7 +3,9 @@ import datetime import json import re +import string import tempfile +import types import warnings from email.utils import parsedate from types import MappingProxyType @@ -16,11 +18,40 @@ from .helpers import HeadersMixin, SimpleCookie, reify, sentinel from .web_exceptions import HTTPRequestEntityTooLarge + __all__ = ('BaseRequest', 'FileField', 'Request') FileField = collections.namedtuple( 'Field', 'name filename file content_type headers') +_TCHAR = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" +# '-' at the end to prevent interpretation as range in a char class + +_TOKEN = r'[{tchar}]*'.format(tchar=_TCHAR) + +_QDTEXT = r'[{}]'.format( + r''.join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F)))) +# qdtext includes 0x5C to escape 0x5D ('\]') +# qdtext excludes obs-text (because obsoleted, and encoding not specified) + +_QUOTED_PAIR = r'\\[\t !-~]' + +_QUOTED_STRING = r'"(?:{quoted_pair}|{qdtext})*"'.format( + qdtext=_QDTEXT, quoted_pair=_QUOTED_PAIR) + +_FORWARDED_PARAMS = ( + r'[bB][yY]|[fF][oO][rR]|[hH][oO][sS][tT]|[pP][rR][oO][tT][oO]') + +_FORWARDED_PAIR = ( + r'^({forwarded_params})=({token}|{quoted_string})$'.format( + forwarded_params=_FORWARDED_PARAMS, + token=_TOKEN, + quoted_string=_QUOTED_STRING)) + +_QUOTED_PAIR_REPLACE_RE = re.compile(r'\\([\t !-~])') +# same pattern as _QUOTED_PAIR but contains a capture group + +_FORWARDED_PAIR_RE = re.compile(_FORWARDED_PAIR) ############################################################ # HTTP Request @@ -150,16 +181,74 @@ def secure(self): """ return self.url.scheme == 'https' + @reify + def forwarded(self): + """ A tuple containing all parsed Forwarded header(s). + + Makes an effort to parse Forwarded headers as specified by RFC 7239: + + - It adds one (immutable) dictionary per Forwarded 'field-value', ie + per proxy. The element corresponds to the data in the Forwarded + field-value added by the first proxy encountered by the client. Each + subsequent item corresponds to those added by later proxies. + - It checks that every value has valid syntax in general as specified + in section 4: either a 'token' or a 'quoted-string'. + - It un-escapes found escape sequences. + - It does NOT validate 'by' and 'for' contents as specified in section + 6. + - It does NOT validate 'host' contents (Host ABNF). + - It does NOT validate 'proto' contents for valid URI scheme names. + + Returns a tuple containing one or more immutable dicts + """ + def _parse_forwarded(forwarded_headers): + for field_value in forwarded_headers: + # by=...;for=..., For=..., BY=... + for forwarded_elm in field_value.split(','): + # by=...;for=... + fvparams = dict() + forwarded_pairs = ( + _FORWARDED_PAIR_RE.findall(pair) + for pair in forwarded_elm.strip().split(';')) + for forwarded_pair in forwarded_pairs: + # by=... + if len(forwarded_pair) != 1: + # non-compliant syntax + break + param, value = forwarded_pair[0] + if param.lower() in fvparams: + # duplicate param in field-value + break + if value and value[0] == '"': + # quoted string: replace quotes and escape + # sequences + value = _QUOTED_PAIR_REPLACE_RE.sub( + r'\1', value[1:-1]) + fvparams[param.lower()] = value + else: + yield types.MappingProxyType(fvparams) + continue + yield dict() + + return tuple( + _parse_forwarded(self._message.headers.getall(hdrs.FORWARDED, ()))) + @reify def _scheme(self): + proto = None if self._transport.get_extra_info('sslcontext'): - return 'https' - secure_proxy_ssl_header = self._secure_proxy_ssl_header - if secure_proxy_ssl_header is not None: - header, value = secure_proxy_ssl_header + proto = 'https' + elif self._secure_proxy_ssl_header is not None: + header, value = self._secure_proxy_ssl_header if self.headers.get(header) == value: - return 'https' - return 'http' + proto = 'https' + else: + proto = next( + (f['proto'] for f in self.forwarded if 'proto' in f), None + ) + if not proto and hdrs.X_FORWARDED_PROTO in self._message.headers: + proto = self._message.headers[hdrs.X_FORWARDED_PROTO] + return proto or 'http' @property def method(self): @@ -179,16 +268,29 @@ def version(self): @reify def host(self): - """Read only property for getting *HOST* header of request. + """ Hostname of the request. + + Hostname is resolved through the following headers, in this order: + + - Forwarded + - X-Forwarded-Host + - Host - Returns str or None if HTTP request has no HOST header. + Returns str, or None if no hostname is found in the headers. """ - return self._message.headers.get(hdrs.HOST) + host = next( + (f['host'] for f in self.forwarded if 'host' in f), None + ) + if not host and hdrs.X_FORWARDED_HOST in self._message.headers: + host = self._message.headers[hdrs.X_FORWARDED_HOST] + elif hdrs.HOST in self._message.headers: + host = self._message.headers[hdrs.HOST] + return host @reify def url(self): return URL('{}://{}{}'.format(self._scheme, - self._message.headers.get(hdrs.HOST), + self.host, str(self._rel_url))) @property diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index f1de7273224..f096c86556a 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -13,6 +13,7 @@ from .helpers import HeadersMixin, SimpleCookie, sentinel from .http import RESPONSES, SERVER_SOFTWARE, HttpVersion10, HttpVersion11 + __all__ = ('ContentCoding', 'StreamResponse', 'Response', 'json_response') diff --git a/aiohttp/web_server.py b/aiohttp/web_server.py index 8e240e2e0c4..f936395df17 100644 --- a/aiohttp/web_server.py +++ b/aiohttp/web_server.py @@ -5,6 +5,7 @@ from .web_protocol import RequestHandler from .web_request import BaseRequest + __all__ = ('Server',) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 0afec9475ba..4e10ce410ee 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -23,12 +23,14 @@ from .web_fileresponse import FileResponse from .web_response import Response, StreamResponse + __all__ = ('UrlDispatcher', 'UrlMappingMatchInfo', 'AbstractResource', 'Resource', 'PlainResource', 'DynamicResource', 'AbstractRoute', 'ResourceRoute', 'StaticResource', 'View') HTTP_METHOD_RE = re.compile(r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$") +PATH_SEP = re.escape('/') class AbstractResource(Sized, Iterable): @@ -328,7 +330,7 @@ class DynamicResource(Resource): def __init__(self, pattern, formatter, *, name=None): super().__init__(name=name) - assert pattern.pattern.startswith('\\/') + assert pattern.pattern.startswith(PATH_SEP) assert formatter.startswith('/') self._pattern = pattern self._formatter = formatter diff --git a/aiohttp/web_ws.py b/aiohttp/web_ws.py index fde267898cf..3f808765535 100644 --- a/aiohttp/web_ws.py +++ b/aiohttp/web_ws.py @@ -5,13 +5,14 @@ from . import hdrs from .helpers import PY_35, PY_352, Timeout, call_later, create_future from .http import (WS_CLOSED_MESSAGE, WS_CLOSING_MESSAGE, HttpProcessingError, - WebSocketError, WebSocketReader, - WSMessage, WSMsgType, do_handshake) + WebSocketError, WebSocketReader, WSMessage, WSMsgType, + do_handshake) from .streams import FlowControlDataQueue from .web_exceptions import (HTTPBadRequest, HTTPInternalServerError, HTTPMethodNotAllowed) from .web_response import StreamResponse + __all__ = ('WebSocketResponse', 'WebSocketReady', 'MsgType', 'WSMsgType',) THRESHOLD_CONNLOST_ACCESS = 5 diff --git a/aiohttp/worker.py b/aiohttp/worker.py index 0ccc4688634..212f1b1bb2c 100644 --- a/aiohttp/worker.py +++ b/aiohttp/worker.py @@ -13,6 +13,7 @@ from .helpers import AccessLogger, create_future, ensure_future + __all__ = ('GunicornWebWorker', 'GunicornUVLoopWebWorker', 'GunicornTokioWebWorker') diff --git a/demos/chat/aiohttpdemo_chat/views.py b/demos/chat/aiohttpdemo_chat/views.py index fc23cc34fc1..f1b0b77fd01 100644 --- a/demos/chat/aiohttpdemo_chat/views.py +++ b/demos/chat/aiohttpdemo_chat/views.py @@ -6,6 +6,7 @@ import aiohttp_jinja2 from aiohttp import web + log = logging.getLogger(__name__) diff --git a/demos/polls/aiohttpdemo_polls/__main__.py b/demos/polls/aiohttpdemo_polls/__main__.py index 1ea11eb3a16..8d5028c2f3b 100644 --- a/demos/polls/aiohttpdemo_polls/__main__.py +++ b/demos/polls/aiohttpdemo_polls/__main__.py @@ -1,4 +1,6 @@ import sys + from aiohttpdemo_polls.main import main + main(sys.argv[1:]) diff --git a/demos/polls/aiohttpdemo_polls/db.py b/demos/polls/aiohttpdemo_polls/db.py index be9fef35351..d4a354abaea 100644 --- a/demos/polls/aiohttpdemo_polls/db.py +++ b/demos/polls/aiohttpdemo_polls/db.py @@ -1,6 +1,6 @@ +import aiopg.sa import sqlalchemy as sa -import aiopg.sa __all__ = ['question', 'choice'] diff --git a/demos/polls/aiohttpdemo_polls/main.py b/demos/polls/aiohttpdemo_polls/main.py index 58c2e31bfa1..2f2dbf00a97 100644 --- a/demos/polls/aiohttpdemo_polls/main.py +++ b/demos/polls/aiohttpdemo_polls/main.py @@ -5,15 +5,13 @@ import jinja2 -from trafaret_config import commandline - - import aiohttp_jinja2 from aiohttp import web from aiohttpdemo_polls.db import close_pg, init_pg from aiohttpdemo_polls.middlewares import setup_middlewares from aiohttpdemo_polls.routes import setup_routes from aiohttpdemo_polls.utils import TRAFARET +from trafaret_config import commandline def init(loop, argv): diff --git a/demos/polls/aiohttpdemo_polls/routes.py b/demos/polls/aiohttpdemo_polls/routes.py index fc74a766689..ad12af3b749 100644 --- a/demos/polls/aiohttpdemo_polls/routes.py +++ b/demos/polls/aiohttpdemo_polls/routes.py @@ -2,6 +2,7 @@ from .views import index, poll, results, vote + PROJECT_ROOT = pathlib.Path(__file__).parent diff --git a/demos/polls/aiohttpdemo_polls/utils.py b/demos/polls/aiohttpdemo_polls/utils.py index 9283dd6cdd4..45688130b65 100644 --- a/demos/polls/aiohttpdemo_polls/utils.py +++ b/demos/polls/aiohttpdemo_polls/utils.py @@ -1,5 +1,6 @@ import trafaret as T + primitive_ip_regexp = r'^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' TRAFARET = T.Dict({ diff --git a/demos/polls/tests/conftest.py b/demos/polls/tests/conftest.py index f0d66a13e72..49c4ff5d343 100644 --- a/demos/polls/tests/conftest.py +++ b/demos/polls/tests/conftest.py @@ -5,6 +5,7 @@ from aiohttpdemo_polls.main import init + BASE_DIR = pathlib.Path(__file__).parent.parent diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 72f355e8b60..21988c6adb4 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -66,8 +66,10 @@ and :ref:`aiohttp-web-signals` handlers. A string representing the scheme of the request. - The scheme is ``'https'`` if transport for request handling is - *SSL* or ``secure_proxy_ssl_header`` is matching. + The scheme is ``'https'`` if transport for request handling is *SSL*, if + ``secure_proxy_ssl_header`` is matching (deprecated), if the ``proto`` + portion of a ``Forward`` header is present and contains ``https``, or if + an ``X-Forwarded-Proto`` header is present and contains ``https``. ``'http'`` otherwise. @@ -81,9 +83,7 @@ and :ref:`aiohttp-web-signals` handlers. .. attribute:: secure - A boolean indicating if transport for request handling is - *SSL* or ``secure_proxy_ssl_header`` is matching, - e.g. if ``request.url.scheme == 'https'`` + Shorthand for ``request.url.scheme == 'https'`` Read-only :class:`bool` property. @@ -1049,7 +1049,7 @@ WebSocketResponse .. seealso:: :ref:`WebSockets handling` - + WebSocketReady ^^^^^^^^^^^^^^ @@ -1233,11 +1233,13 @@ duplicated like one using :meth:`Application.copy`. If param is ``None`` :func:`asyncio.get_event_loop` used for getting default event loop. - :param tuple secure_proxy_ssl_header: Secure proxy SSL header. Can - be used to detect request scheme, - e.g. ``secure_proxy_ssl_header=('X-Forwarded-Proto', 'https')``. + :param tuple secure_proxy_ssl_header: Default: ``None``. + + .. deprecated:: 2.1 + + See ``request.url.scheme`` for built-in resolution of the current + scheme using the standard and de-facto standard headers. - Default: ``None``. :param bool tcp_keepalive: Enable TCP Keep-Alive. Default: ``True``. :param int keepalive_timeout: Number of seconds before closing Keep-Alive connection. Default: ``75`` seconds (NGINX's default value). diff --git a/examples/client_ws.py b/examples/client_ws.py index d054dec3482..233e6d45aa9 100755 --- a/examples/client_ws.py +++ b/examples/client_ws.py @@ -7,6 +7,7 @@ import aiohttp + try: import selectors except ImportError: diff --git a/examples/legacy/srv.py b/examples/legacy/srv.py index 0941273db6d..a5245cf1a35 100755 --- a/examples/legacy/srv.py +++ b/examples/legacy/srv.py @@ -10,6 +10,7 @@ import aiohttp import aiohttp.server + try: import ssl except ImportError: # pragma: no cover diff --git a/examples/legacy/tcp_protocol_parser.py b/examples/legacy/tcp_protocol_parser.py index a9e2df096dd..a8766042c68 100755 --- a/examples/legacy/tcp_protocol_parser.py +++ b/examples/legacy/tcp_protocol_parser.py @@ -6,6 +6,7 @@ import aiohttp + try: import signal except ImportError: diff --git a/examples/static_files.py b/examples/static_files.py index 426242a8514..4ffd1ab60e1 100755 --- a/examples/static_files.py +++ b/examples/static_files.py @@ -2,6 +2,7 @@ from aiohttp import web + app = web.Application() app.router.add_static('/', pathlib.Path(__file__).parent, show_index=True) diff --git a/examples/web_ws.py b/examples/web_ws.py index 154a3ca4e0d..7660f931ea3 100755 --- a/examples/web_ws.py +++ b/examples/web_ws.py @@ -8,6 +8,7 @@ from aiohttp.web import (Application, Response, WebSocketResponse, WSMsgType, run_app) + WS_FILE = os.path.join(os.path.dirname(__file__), 'websocket.html') diff --git a/tests/conftest.py b/tests/conftest.py index 2cfc3bdd0ca..429fc743705 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = 'aiohttp.pytest_plugin' diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 426af2e0df0..6fe2b026ad4 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -14,6 +14,7 @@ from aiohttp.http_parser import (DeflateBuffer, HttpPayloadParser, HttpRequestParserPy, HttpResponseParserPy) + REQUEST_PARSERS = [HttpRequestParserPy] RESPONSE_PARSERS = [HttpResponseParserPy] diff --git a/tests/test_http_stream_writer.py b/tests/test_http_stream_writer.py index 1ba15cd72a2..bf5ef8be280 100644 --- a/tests/test_http_stream_writer.py +++ b/tests/test_http_stream_writer.py @@ -5,6 +5,7 @@ from aiohttp.http_writer import CORK, PayloadWriter, StreamWriter + has_ipv6 = socket.has_ipv6 if has_ipv6: # The socket.has_ipv6 flag may be True if Python was built with IPv6 diff --git a/tests/test_py35/test_streams_35.py b/tests/test_py35/test_streams_35.py index ef25bcca52d..1644e230708 100644 --- a/tests/test_py35/test_streams_35.py +++ b/tests/test_py35/test_streams_35.py @@ -2,6 +2,7 @@ from aiohttp import streams + DATA = b'line1\nline2\nline3\n' diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 9001f38d35d..9b6665d1f47 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -1,7 +1,9 @@ import re import sys + import pytest + pytest_plugins = 'pytester' diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 713134b3fbf..b824bdbb412 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -7,6 +7,7 @@ from aiohttp.resolver import AsyncResolver, DefaultResolver, ThreadedResolver + try: import aiodns gethostbyname = hasattr(aiodns.DNSResolver, 'gethostbyname') diff --git a/tests/test_run_app.py b/tests/test_run_app.py index e3f37775aba..13e4db0ba90 100644 --- a/tests/test_run_app.py +++ b/tests/test_run_app.py @@ -3,7 +3,6 @@ import os import socket import ssl - from io import StringIO from unittest import mock from uuid import uuid4 diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index 49821dbf5ee..a6a9764602d 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -12,8 +12,8 @@ from aiohttp import hdrs, web from aiohttp.test_utils import make_mocked_request from aiohttp.web import HTTPMethodNotAllowed, HTTPNotFound, Response -from aiohttp.web_urldispatcher import (AbstractResource, ResourceRoute, - SystemRoute, View, +from aiohttp.web_urldispatcher import (PATH_SEP, AbstractResource, + ResourceRoute, SystemRoute, View, _defaultExpectHandler) @@ -495,8 +495,11 @@ def test_add_route_with_invalid_re(router): with pytest.raises(ValueError) as ctx: router.add_route('GET', r'/handler/{to:+++}', handler) s = str(ctx.value) - assert s.startswith( - "Bad pattern '\/handler\/(?P+++)': nothing to repeat") + assert s.startswith("Bad pattern '" + + PATH_SEP + + "handler" + + PATH_SEP + + "(?P+++)': nothing to repeat") assert ctx.value.__cause__ is None @@ -798,7 +801,7 @@ def test_match_info_get_info_dynamic(router): req = make_request('GET', '/value') info = yield from router.resolve(req) assert info.get_info() == { - 'pattern': re.compile('\\/(?P[^{}/]+)'), + 'pattern': re.compile(PATH_SEP+'(?P[^{}/]+)'), 'formatter': '/{a}'} @@ -809,7 +812,10 @@ def test_match_info_get_info_dynamic2(router): req = make_request('GET', '/path/to') info = yield from router.resolve(req) assert info.get_info() == { - 'pattern': re.compile('\\/(?P[^{}/]+)\\/(?P[^{}/]+)'), + 'pattern': re.compile(PATH_SEP + + '(?P[^{}/]+)' + + PATH_SEP + + '(?P[^{}/]+)'), 'formatter': '/{a}/{b}'} diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 94f07b66c8c..3f673c2ce9b 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -12,6 +12,7 @@ import aiohttp from aiohttp import FormData, HttpVersion10, HttpVersion11, multipart, web + try: import ssl except: diff --git a/tests/test_web_request.py b/tests/test_web_request.py index 2f3fbe4462f..440cfd71e59 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -248,6 +248,116 @@ def test_https_scheme_by_secure_proxy_ssl_header_false_test(make_request): assert req.secure is False +def test_single_forwarded_header(make_request): + header = 'by=identifier;for=identifier;host=identifier;proto=identifier' + req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['by'] == 'identifier' + assert req.forwarded[0]['for'] == 'identifier' + assert req.forwarded[0]['host'] == 'identifier' + assert req.forwarded[0]['proto'] == 'identifier' + + +def test_single_forwarded_header_camelcase(make_request): + header = 'bY=identifier;fOr=identifier;HOst=identifier;pRoTO=identifier' + req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['by'] == 'identifier' + assert req.forwarded[0]['for'] == 'identifier' + assert req.forwarded[0]['host'] == 'identifier' + assert req.forwarded[0]['proto'] == 'identifier' + + +def test_single_forwarded_header_single_param(make_request): + header = 'BY=identifier' + req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['by'] == 'identifier' + + +def test_single_forwarded_header_multiple_param(make_request): + header = 'By=identifier1,BY=identifier2, By=identifier3 , BY=identifier4' + req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + assert len(req.forwarded) == 4 + assert req.forwarded[0]['by'] == 'identifier1' + assert req.forwarded[1]['by'] == 'identifier2' + assert req.forwarded[2]['by'] == 'identifier3' + assert req.forwarded[3]['by'] == 'identifier4' + + +def test_single_forwarded_header_quoted_escaped(make_request): + header = 'BY=identifier;pROTO="\lala lan\d\~ 123\!&"' + req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['by'] == 'identifier' + assert req.forwarded[0]['proto'] == 'lala land~ 123!&' + + +def test_multiple_forwarded_headers(make_request): + headers = CIMultiDict() + headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') + headers.add('Forwarded', 'By=identifier4;fOr=identifier5') + req = make_request('GET', '/', headers=headers) + assert len(req.forwarded) == 3 + assert req.forwarded[0]['by'] == 'identifier1' + assert req.forwarded[0]['for'] == 'identifier2' + assert req.forwarded[1]['by'] == 'identifier3' + assert req.forwarded[2]['by'] == 'identifier4' + assert req.forwarded[2]['for'] == 'identifier5' + + +def test_https_scheme_by_forwarded_header(make_request): + req = make_request('GET', '/', + headers=CIMultiDict( + {'Forwarded': 'by=;for=;host=;proto=https'})) + assert "https" == req.scheme + assert req.secure is True + + +def test_https_scheme_by_malformed_forwarded_header(make_request): + req = make_request('GET', '/', + headers=CIMultiDict({'Forwarded': 'malformed value'})) + assert "http" == req.scheme + assert req.secure is False + + +def test_https_scheme_by_x_forwarded_proto_header(make_request): + req = make_request('GET', '/', + headers=CIMultiDict({'X-Forwarded-Proto': 'https'})) + assert "https" == req.scheme + assert req.secure is True + + +def test_https_scheme_by_x_forwarded_proto_header_no_tls(make_request): + req = make_request('GET', '/', + headers=CIMultiDict({'X-Forwarded-Proto': 'http'})) + assert "http" == req.scheme + assert req.secure is False + + +def test_host_by_forwarded_header(make_request): + headers = CIMultiDict() + headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') + headers.add('Forwarded', 'by=;for=;host=example.com') + req = make_request('GET', '/', headers=headers) + assert req.host == 'example.com' + + +def test_host_by_forwarded_header_malformed(make_request): + req = make_request('GET', '/', + headers=CIMultiDict({'Forwarded': 'malformed value'})) + assert req.host is None + + +def test_host_by_x_forwarded_host_header(make_request): + req = make_request('GET', '/', + headers=CIMultiDict( + {'X-Forwarded-Host': 'example.com'})) + assert req.host == 'example.com' + + +def test_host_by_host_header(make_request): + req = make_request('GET', '/', + headers=CIMultiDict({'Host': 'example.com'})) + assert req.host == 'example.com' + + def test_raw_headers(make_request): req = make_request('GET', '/', headers=CIMultiDict({'X-HEADER': 'aaa'})) diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index fdea28bde64..caecfd9dc91 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -7,6 +7,7 @@ import aiohttp from aiohttp import web + try: import ssl except: diff --git a/tests/test_worker.py b/tests/test_worker.py index 252e937c58a..88f0b4a4d85 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -10,6 +10,7 @@ from aiohttp import helpers from aiohttp.test_utils import make_mocked_coro + base_worker = pytest.importorskip('aiohttp.worker')