Merge pull request #415 from KeepSafe/connector_caching

Connector caching

aiohttp/connector.py

@@ -384,6 +384,8 @@ def _create_connection(self, req):
 _SSL_OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0)
 _SSH_HAS_CREATE_DEFAULT_CONTEXT = hasattr(ssl, 'create_default_context')
 
+_marker = object()
+
 
 class TCPConnector(BaseConnector):
     """TCP connector.
@@ -400,7 +402,8 @@ class TCPConnector(BaseConnector):
     """
 
     def __init__(self, *, verify_ssl=True, fingerprint=None,
-                 resolve=False, family=socket.AF_INET, ssl_context=None,
+                 resolve=_marker, use_dns_cache=_marker,
+                 family=socket.AF_INET, ssl_context=None,
                  **kwargs):
         super().__init__(**kwargs)
 
@@ -419,10 +422,26 @@ def __init__(self, *, verify_ssl=True, fingerprint=None,
             self._hashfunc = hashfunc
         self._fingerprint = fingerprint
 
+        if resolve is not _marker:
+            warnings.warn(("resolve parameter is deprecated, "
+                           "use use_dns_cache instead"),
+                          DeprecationWarning, stacklevel=2)
+
+        if use_dns_cache is not _marker and resolve is not _marker:
+            if use_dns_cache != resolve:
+                raise ValueError("use_dns_cache must agree with resolve")
+            _use_dns_cache = use_dns_cache
+        elif use_dns_cache is not _marker:
+            _use_dns_cache = use_dns_cache
+        elif resolve is not _marker:
+            _use_dns_cache = resolve
+        else:
+            _use_dns_cache = False
+
+        self._use_dns_cache = _use_dns_cache
+        self._cached_hosts = {}
         self._ssl_context = ssl_context
         self._family = family
-        self._resolve = resolve
-        self._resolved_hosts = {}
 
     @property
     def verify_ssl(self):
@@ -466,31 +485,58 @@ def family(self):
         """Socket family like AF_INET."""
         return self._family
 
+    @property
+    def use_dns_cache(self):
+        """True if local DNS caching is enabled."""
+        return self._use_dns_cache
+
+    @property
+    def cached_hosts(self):
+        """Read-only dict of cached DNS record."""
+        return MappingProxyType(self._cached_hosts)
+
+    def clear_dns_cache(self, host=None, port=None):
+        """Remove specified host/port or clear all dns local cache."""
+        if host is not None and port is not None:
+            self._cached_hosts.pop((host, port), None)
+        elif host is not None or port is not None:
+            raise ValueError("either both host and port "
+                             "or none of them are allowed")
+        else:
+            self._cached_hosts.clear()
+
     @property
     def resolve(self):
         """Do DNS lookup for host name?"""
-        return self._resolve
+        warnings.warn((".resolve property is deprecated, "
+                       "use .dns_cache instead"),
+                      DeprecationWarning, stacklevel=2)
+        return self.use_dns_cache
 
     @property
     def resolved_hosts(self):
         """The dict of (host, port) -> (ipaddr, port) pairs."""
-        return MappingProxyType(self._resolved_hosts)
+        warnings.warn((".resolved_hosts property is deprecated, "
+                       "use .cached_hosts instead"),
+                      DeprecationWarning, stacklevel=2)
+        return self.cached_hosts
 
     def clear_resolved_hosts(self, host=None, port=None):
         """Remove specified host/port or clear all resolve cache."""
+        warnings.warn((".clear_resolved_hosts() is deprecated, "
+                       "use .clear_dns_cache() instead"),
+                      DeprecationWarning, stacklevel=2)
         if host is not None and port is not None:
-            key = (host, port)
-            if key in self._resolved_hosts:
-                del self._resolved_hosts[key]
+            self.clear_dns_cache(host, port)
         else:
-            self._resolved_hosts.clear()
+            self.clear_dns_cache()
 
     @asyncio.coroutine
     def _resolve_host(self, host, port):
-        if self._resolve:
+        if self._use_dns_cache:
             key = (host, port)
 
-            if key not in self._resolved_hosts:
+            if key not in self._cached_hosts:
                 infos = yield from self._loop.getaddrinfo(
                     host, port, type=socket.SOCK_STREAM, family=self._family)
 
@@ -501,9 +547,9 @@ def _resolve_host(self, host, port):
                          'host': address[0], 'port': address[1],
                          'family': family, 'proto': proto,
                          'flags': socket.AI_NUMERICHOST})
-                self._resolved_hosts[key] = hosts
+                self._cached_hosts[key] = hosts
 
-            return list(self._resolved_hosts[key])
+            return list(self._cached_hosts[key])
         else:
             return [{'hostname': host, 'host': host, 'port': port,
                      'family': self._family, 'proto': 0, 'flags': 0}]

docs/client_reference.rst

@@ -464,7 +464,7 @@ BaseConnector
 TCPConnector
 ^^^^^^^^^^^^
 
-.. class:: TCPConnector(*, verify_ssl=True, fingerprint=None, resolve=False, \
+.. class:: TCPConnector(*, verify_ssl=True, fingerprint=None, use_dns_cache=False, \
                         family=socket.AF_INET, \
                         ssl_context=None, conn_timeout=None, \
                         keepalive_timeout=30, limit=None, share_cookies=False, \
@@ -492,13 +492,19 @@ TCPConnector
 
         .. versionadded:: 0.16
 
-   :param bool resolve: use internal cache for DNS lookups, ``False``
+   :param bool use_dns_cache: use internal cache for DNS lookups, ``False``
       by default.
 
       Enabling an option *may* speedup connection
       establishing a bit but may introduce some
       *side effects* also.
 
+      .. versionadded:: 0.17
+
+   :param bool resolve: alias for *use_dns_cache* parameter.
+
+      .. deprecated:: 0.17
+
    :param int family: TCP socket family, ``AF_INET`` by default
                       (*IPv4*). For *IPv6* use ``AF_INET6``.
 
@@ -525,24 +531,34 @@ TCPConnector
 
       Read-only property.
 
-   .. attribute:: resolve
+   .. attribute:: dns_cache
 
       Use quick lookup in internal *DNS* cache for host names if ``True``.
 
       Read-only :class:`bool` property.
 
+      .. versionadded:: 0.17
+
    .. attribute:: resolve
 
-      Use quick lookup in internal *DNS* cache for host names if ``True``.
+      Alias for :attr:`dns_cache`.
 
-      Read-only :class:`bool` property.
+      .. deprecated:: 0.17
 
-   .. attribute:: resolved_hosts
+   .. attribute:: cached_hosts
 
-      The cache of resolved hosts if :attr:`resolve` is enabled.
+      The cache of resolved hosts if :attr:`dns_cache` is enabled.
 
       Read-only :class:`types.MappingProxyType` property.
 
+      .. versionadded:: 0.17
+
+   .. attribute:: resolved_hosts
+
+      Alias for :attr:`cached_hosts`
+
+      .. deprecated:: 0.17
+
    .. attribute:: fingerprint
 
       md5, sha1, or sha256 hash of the expected certificate in DER
@@ -553,13 +569,21 @@ TCPConnector
 
       .. versionadded:: 0.16
 
-   .. method:: clear_resolved_hosts(self, host=None, port=None)
+   .. method:: clear_dns_cache(self, host=None, port=None)
 
       Clear internal *DNS* cache.
 
       Remove specific entry if both *host* and *port* are specified,
       clear all cache otherwise.
 
+      .. versionadded:: 0.17
+
+   .. method:: clear_resolved_hosts(self, host=None, port=None)
+
+      Alias for :meth:`clear_dns_cache`.
+
+      .. deprecated:: 0.17
+
 
 
 

tests/test_connector.py

@@ -458,8 +458,15 @@ def test_tcp_connector_ctor(self):
         conn = aiohttp.TCPConnector(loop=self.loop)
         self.assertTrue(conn.verify_ssl)
         self.assertIs(conn.fingerprint, None)
-        self.assertFalse(conn.resolve)
+
+        with self.assertWarns(DeprecationWarning):
+            self.assertFalse(conn.resolve)
+        self.assertFalse(conn.use_dns_cache)
+
         self.assertEqual(conn.family, socket.AF_INET)
+
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(conn.resolved_hosts, {})
         self.assertEqual(conn.resolved_hosts, {})
 
     def test_tcp_connector_ctor_fingerprint_valid(self):
@@ -514,17 +521,37 @@ def test_tcp_connector_fingerprint(self):
     def test_tcp_connector_clear_resolved_hosts(self):
         conn = aiohttp.TCPConnector(loop=self.loop)
         info = object()
-        conn._resolved_hosts[('localhost', 123)] = info
-        conn._resolved_hosts[('localhost', 124)] = info
+        conn._cached_hosts[('localhost', 123)] = info
+        conn._cached_hosts[('localhost', 124)] = info
         conn.clear_resolved_hosts('localhost', 123)
         self.assertEqual(
             conn.resolved_hosts, {('localhost', 124): info})
         conn.clear_resolved_hosts('localhost', 123)
         self.assertEqual(
             conn.resolved_hosts, {('localhost', 124): info})
-        conn.clear_resolved_hosts()
+        with self.assertWarns(DeprecationWarning):
+            conn.clear_resolved_hosts()
         self.assertEqual(conn.resolved_hosts, {})
 
+    def test_tcp_connector_clear_dns_cache(self):
+        conn = aiohttp.TCPConnector(loop=self.loop)
+        info = object()
+        conn._cached_hosts[('localhost', 123)] = info
+        conn._cached_hosts[('localhost', 124)] = info
+        conn.clear_dns_cache('localhost', 123)
+        self.assertEqual(
+            conn.cached_hosts, {('localhost', 124): info})
+        conn.clear_dns_cache('localhost', 123)
+        self.assertEqual(
+            conn.cached_hosts, {('localhost', 124): info})
+        conn.clear_dns_cache()
+        self.assertEqual(conn.cached_hosts, {})
+
+    def test_tcp_connector_clear_dns_cache_bad_args(self):
+        conn = aiohttp.TCPConnector(loop=self.loop)
+        with self.assertRaises(ValueError):
+            conn.clear_dns_cache('localhost')
+
     def test_ambigous_verify_ssl_and_ssl_context(self):
         with self.assertRaises(ValueError):
             aiohttp.TCPConnector(
@@ -756,6 +783,25 @@ def test_connector_cookie_deprecation(self):
             conn = aiohttp.TCPConnector(share_cookies=True, loop=self.loop)
         conn.close()
 
+    def test_ambiguous_ctor_params(self):
+        with self.assertRaises(ValueError):
+            aiohttp.TCPConnector(resolve=True, use_dns_cache=False,
+                                 loop=self.loop)
+
+    def test_both_resolve_and_use_dns_cache(self):
+        conn = aiohttp.TCPConnector(resolve=True, use_dns_cache=True,
+                                    loop=self.loop)
+        self.assertTrue(conn.use_dns_cache)
+        with self.assertWarns(DeprecationWarning):
+            self.assertTrue(conn.resolve)
+
+    def test_both_use_dns_cache_only(self):
+        conn = aiohttp.TCPConnector(use_dns_cache=True,
+                                    loop=self.loop)
+        self.assertTrue(conn.use_dns_cache)
+        with self.assertWarns(DeprecationWarning):
+            self.assertTrue(conn.resolve)
+
 
 class TestProxyConnector(unittest.TestCase):