Merge pull request #472 from elastic-coders/optimize_acquired_list

Improve performance of transport in-use list.

aiohttp/connector.py

@@ -122,7 +122,7 @@ def __init__(self, *, conn_timeout=None, keepalive_timeout=30,
             self._source_traceback = traceback.extract_stack(sys._getframe(1))
 
         self._conns = {}
-        self._acquired = defaultdict(list)
+        self._acquired = defaultdict(set)
         self._conn_timeout = conn_timeout
         self._keepalive_timeout = keepalive_timeout
         if share_cookies:
@@ -304,7 +304,7 @@ def connect(self, req):
                 raise ClientOSError(
                     'Cannot connect to host %s:%s ssl:%s' % key) from exc
 
-        self._acquired[key].append(transport)
+        self._acquired[key].add(transport)
         conn = Connection(self, key, req, transport, proto, self._loop)
         return conn
 
@@ -330,7 +330,7 @@ def _release(self, key, req, transport, protocol, *, should_close=False):
         acquired = self._acquired[key]
         try:
             acquired.remove(transport)
-        except ValueError:  # pragma: no cover
+        except KeyError:  # pragma: no cover
             # this may be result of undetermenistic order of objects
             # finalization due garbage collection.
             pass
@@ -672,7 +672,7 @@ def _create_connection(self, req):
             key = (req.host, req.port, req.ssl)
             conn = Connection(self, key, proxy_req,
                               transport, proto, self._loop)
-            self._acquired[key].append(conn._transport)
+            self._acquired[key].add(conn._transport)
             proxy_resp = proxy_req.send(conn.writer, conn.reader)
             try:
                 resp = yield from proxy_resp.start(conn, True)

tests/test_connector.py

@@ -253,7 +253,7 @@ def test_release(self):
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
         key = 1
-        conn._acquired[key].append(tr)
+        conn._acquired[key].add(tr)
         conn._release(key, req, tr, proto)
         self.assertEqual(conn._conns[1][0], (tr, proto, 10))
         self.assertTrue(conn._start_cleanup_task.called)
@@ -273,7 +273,7 @@ def test_release_close(self):
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
         key = 1
-        conn._acquired[key].append(tr)
+        conn._acquired[key].add(tr)
         conn._release(key, req, tr, proto)
         self.assertFalse(conn._conns)
         self.assertTrue(tr.close.called)
@@ -291,7 +291,7 @@ def test_release_pop_empty_conns(self):
         conn._conns[key] = []
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
-        conn._acquired[key].append(tr)
+        conn._acquired[key].add(tr)
         conn._release(key, req, tr, proto)
         self.assertEqual({}, conn._conns)
         self.assertTrue(tr.close.called)
@@ -309,7 +309,7 @@ def test_release_close_do_not_delete_existing_connections(self):
         req.response = resp
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
-        conn._acquired[key].append(tr1)
+        conn._acquired[key].add(tr1)
         conn._release(key, req, tr, proto)
         self.assertEqual(conn._conns[key], [(tr1, proto1, 1)])
         self.assertTrue(tr.close.called)
@@ -324,7 +324,7 @@ def test_release_not_started(self):
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
         key = 1
-        conn._acquired[key].append(tr)
+        conn._acquired[key].add(tr)
         conn._release(key, req, tr, proto)
         self.assertEqual(conn._conns, {1: [(tr, proto, 10)]})
         self.assertFalse(tr.close.called)
@@ -338,7 +338,7 @@ def test_release_not_opened(self):
 
         tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
         key = 1
-        conn._acquired[key].append(tr)
+        conn._acquired[key].add(tr)
         conn._release(key, req, tr, proto)
         self.assertTrue(tr.close.called)