From bd223cac60305d4f65b52376b5d927a7ec246319 Mon Sep 17 00:00:00 2001 From: Mark Keller Date: Fri, 3 Nov 2023 17:03:34 -0700 Subject: [PATCH 1/4] updating vendored requests to 2.31.0 --- setup.cfg | 1 - src/snowflake/connector/vendored/requests/__init__.py | 8 ++++---- src/snowflake/connector/vendored/requests/__version__.py | 4 ++-- src/snowflake/connector/vendored/requests/adapters.py | 1 - src/snowflake/connector/vendored/requests/sessions.py | 4 +++- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2836f2087..a5381313b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,7 +53,6 @@ install_requires = packaging charset_normalizer>=2,<4 idna>=2.5,<4 - urllib3>=1.21.1,<1.27 certifi>=2017.4.17 typing_extensions>=4.3,<5 filelock>=3.5,<4 diff --git a/src/snowflake/connector/vendored/requests/__init__.py b/src/snowflake/connector/vendored/requests/__init__.py index 67fdaa8c2..03c3f69d3 100644 --- a/src/snowflake/connector/vendored/requests/__init__.py +++ b/src/snowflake/connector/vendored/requests/__init__.py @@ -66,10 +66,10 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.26 - assert major == 1 - assert minor >= 21 - assert minor <= 26 + # urllib3 >= 1.21.1 + assert major >= 1 + if major == 1: + assert minor >= 21 # Check charset_normalizer for compatibility. if chardet_version: diff --git a/src/snowflake/connector/vendored/requests/__version__.py b/src/snowflake/connector/vendored/requests/__version__.py index 4775ae32e..5063c3f8e 100644 --- a/src/snowflake/connector/vendored/requests/__version__.py +++ b/src/snowflake/connector/vendored/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = "requests" __description__ = "Python HTTP for Humans." __url__ = "https://requests.readthedocs.io" -__version__ = "2.29.0" -__build__ = 0x022900 +__version__ = "2.31.0" +__build__ = 0x023100 __author__ = "Kenneth Reitz" __author_email__ = "me@kennethreitz.org" __license__ = "Apache 2.0" diff --git a/src/snowflake/connector/vendored/requests/adapters.py b/src/snowflake/connector/vendored/requests/adapters.py index 303a3402b..ab92194fb 100644 --- a/src/snowflake/connector/vendored/requests/adapters.py +++ b/src/snowflake/connector/vendored/requests/adapters.py @@ -193,7 +193,6 @@ def init_poolmanager( num_pools=connections, maxsize=maxsize, block=block, - strict=True, **pool_kwargs, ) diff --git a/src/snowflake/connector/vendored/requests/sessions.py b/src/snowflake/connector/vendored/requests/sessions.py index 6cb3b4dae..dbcf2a7b0 100644 --- a/src/snowflake/connector/vendored/requests/sessions.py +++ b/src/snowflake/connector/vendored/requests/sessions.py @@ -324,7 +324,9 @@ def rebuild_proxies(self, prepared_request, proxies): except KeyError: username, password = None, None - if username and password: + # urllib3 handles proxy authorization for us in the standard adapter. + # Avoid appending this to TLS tunneled requests where it may be leaked. + if not scheme.startswith('https') and username and password: headers["Proxy-Authorization"] = _basic_auth_str(username, password) return new_proxies From cbcf778039d8279d0ea46369e8cb657eb55ff3ec Mon Sep 17 00:00:00 2001 From: Mark Keller Date: Fri, 3 Nov 2023 17:11:50 -0700 Subject: [PATCH 2/4] updating vendored urllib3 to 1.26.18 --- .../vendored/urllib3/_collections.py | 18 ++ .../connector/vendored/urllib3/_version.py | 2 +- .../vendored/urllib3/connectionpool.py | 43 ++++- .../urllib3/contrib/securetransport.py | 3 +- .../packages/backports/weakref_finalize.py | 155 ++++++++++++++++++ .../connector/vendored/urllib3/poolmanager.py | 9 +- .../connector/vendored/urllib3/request.py | 21 +++ .../connector/vendored/urllib3/util/retry.py | 2 +- 8 files changed, 238 insertions(+), 15 deletions(-) create mode 100644 src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py diff --git a/src/snowflake/connector/vendored/urllib3/_collections.py b/src/snowflake/connector/vendored/urllib3/_collections.py index da9857e98..bceb8451f 100644 --- a/src/snowflake/connector/vendored/urllib3/_collections.py +++ b/src/snowflake/connector/vendored/urllib3/_collections.py @@ -268,6 +268,24 @@ def getlist(self, key, default=__marker): else: return vals[1:] + def _prepare_for_method_change(self): + """ + Remove content-specific header fields before changing the request + method to GET or HEAD according to RFC 9110, Section 15.4. + """ + content_specific_headers = [ + "Content-Encoding", + "Content-Language", + "Content-Location", + "Content-Type", + "Content-Length", + "Digest", + "Last-Modified", + ] + for header in content_specific_headers: + self.discard(header) + return self + # Backwards compatibility for httplib getheaders = getlist getallmatchingheaders = getlist diff --git a/src/snowflake/connector/vendored/urllib3/_version.py b/src/snowflake/connector/vendored/urllib3/_version.py index e12dd0e78..85e725eaf 100644 --- a/src/snowflake/connector/vendored/urllib3/_version.py +++ b/src/snowflake/connector/vendored/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.15" +__version__ = "1.26.18" diff --git a/src/snowflake/connector/vendored/urllib3/connectionpool.py b/src/snowflake/connector/vendored/urllib3/connectionpool.py index c23d736b1..5a6adcbdc 100644 --- a/src/snowflake/connector/vendored/urllib3/connectionpool.py +++ b/src/snowflake/connector/vendored/urllib3/connectionpool.py @@ -9,6 +9,7 @@ from socket import error as SocketError from socket import timeout as SocketTimeout +from ._collections import HTTPHeaderDict from .connection import ( BaseSSLError, BrokenPipeError, @@ -50,6 +51,13 @@ from .util.url import _normalize_host as normalize_host from .util.url import get_host, parse_url +try: # Platform-specific: Python 3 + import weakref + + weakref_finalize = weakref.finalize +except AttributeError: # Platform-specific: Python 2 + from .packages.backports.weakref_finalize import weakref_finalize + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -220,6 +228,16 @@ def __init__( self.conn_kw["proxy"] = self.proxy self.conn_kw["proxy_config"] = self.proxy_config + # Do not pass 'self' as callback to 'finalize'. + # Then the 'finalize' would keep an endless living (leak) to self. + # By just passing a reference to the pool allows the garbage collector + # to free self if nobody else has a reference to it. + pool = self.pool + + # Close all the HTTPConnections in the pool before the + # HTTPConnectionPool object is garbage collected. + weakref_finalize(self, _close_pool_connections, pool) + def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. @@ -489,14 +507,8 @@ def close(self): # Disable access to the pool old_pool, self.pool = self.pool, None - try: - while True: - conn = old_pool.get(block=False) - if conn: - conn.close() - - except queue.Empty: - pass # Done. + # Close all the HTTPConnections in the pool. + _close_pool_connections(old_pool) def is_same_host(self, url): """ @@ -832,7 +844,11 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + body = None + headers = HTTPHeaderDict(headers)._prepare_for_method_change() try: retries = retries.increment(method, url, response=response, _pool=self) @@ -1108,3 +1124,14 @@ def _normalize_host(host, scheme): if host.startswith("[") and host.endswith("]"): host = host[1:-1] return host + + +def _close_pool_connections(pool): + """Drains a queue of connections and closes each one.""" + try: + while True: + conn = pool.get(block=False) + if conn: + conn.close() + except queue.Empty: + pass # Done. diff --git a/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py b/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py index 6c46a3b9f..e311c0c89 100644 --- a/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py +++ b/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py @@ -64,9 +64,8 @@ import threading import weakref -import six - from .. import util +from ..packages import six from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( diff --git a/src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py b/src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py new file mode 100644 index 000000000..a2f2966e5 --- /dev/null +++ b/src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" +backports.weakref_finalize +~~~~~~~~~~~~~~~~~~ + +Backports the Python 3 ``weakref.finalize`` method. +""" +from __future__ import absolute_import + +import itertools +import sys +from weakref import ref + +__all__ = ["weakref_finalize"] + + +class weakref_finalize(object): + """Class for finalization of weakrefable objects + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info(object): + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + + atexit.register(self._exitfunc) + weakref_finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + weakref_finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return "<%s object at %#x; dead>" % (type(self).__name__, id(self)) + else: + return "<%s object at %#x; for %r at %#x>" % ( + type(self).__name__, + id(self), + type(obj).__name__, + id(obj), + ) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f, i) for (f, i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item: item[1].index) + return [f for (f, i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or weakref_finalize._dirty: + pending = cls._select_for_exit() + weakref_finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + weakref_finalize._shutdown = True + if reenable_gc: + gc.enable() diff --git a/src/snowflake/connector/vendored/urllib3/poolmanager.py b/src/snowflake/connector/vendored/urllib3/poolmanager.py index ca4ec3411..fb51bf7d9 100644 --- a/src/snowflake/connector/vendored/urllib3/poolmanager.py +++ b/src/snowflake/connector/vendored/urllib3/poolmanager.py @@ -4,7 +4,7 @@ import functools import logging -from ._collections import RecentlyUsedContainer +from ._collections import HTTPHeaderDict, RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, @@ -171,7 +171,7 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer(num_pools) # Locally set the pool classes and keys so other PoolManagers can # override them. @@ -382,9 +382,12 @@ def urlopen(self, method, url, redirect=True, **kw): # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) - # RFC 7231, Section 6.4.4 if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + kw["body"] = None + kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() retries = kw.get("retries") if not isinstance(retries, Retry): diff --git a/src/snowflake/connector/vendored/urllib3/request.py b/src/snowflake/connector/vendored/urllib3/request.py index 398386a5b..3b4cf9992 100644 --- a/src/snowflake/connector/vendored/urllib3/request.py +++ b/src/snowflake/connector/vendored/urllib3/request.py @@ -1,6 +1,9 @@ from __future__ import absolute_import +import sys + from .filepost import encode_multipart_formdata +from .packages import six from .packages.six.moves.urllib.parse import urlencode __all__ = ["RequestMethods"] @@ -168,3 +171,21 @@ def request_encode_body( extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) + + +if not six.PY2: + + class RequestModule(sys.modules[__name__].__class__): + def __call__(self, *args, **kwargs): + """ + If user tries to call this module directly urllib3 v2.x style raise an error to the user + suggesting they may need urllib3 v2 + """ + raise TypeError( + "'module' object is not callable\n" + "urllib3.request() method is not supported in this release, " + "upgrade to urllib3 v2 to use it\n" + "see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html" + ) + + sys.modules[__name__].__class__ = RequestModule diff --git a/src/snowflake/connector/vendored/urllib3/util/retry.py b/src/snowflake/connector/vendored/urllib3/util/retry.py index 2490d5e5b..60ef6c4f3 100644 --- a/src/snowflake/connector/vendored/urllib3/util/retry.py +++ b/src/snowflake/connector/vendored/urllib3/util/retry.py @@ -235,7 +235,7 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) #: Maximum backoff time. DEFAULT_BACKOFF_MAX = 120 From 516d2b00ebf75821785749b3b592f0bf4db20894 Mon Sep 17 00:00:00 2001 From: Mark Keller Date: Mon, 6 Nov 2023 09:45:34 -0800 Subject: [PATCH 3/4] putting back urllib3 pin --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a5381313b..99085bb53 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,7 @@ install_requires = packaging charset_normalizer>=2,<4 idna>=2.5,<4 + urllib3>=1.21.1,<2.0.0 certifi>=2017.4.17 typing_extensions>=4.3,<5 filelock>=3.5,<4 From 0932367e6c9677a2d73271bc096b1cdd94918717 Mon Sep 17 00:00:00 2001 From: Mark Keller Date: Mon, 6 Nov 2023 11:44:11 -0800 Subject: [PATCH 4/4] adding changelog entry --- DESCRIPTION.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DESCRIPTION.md b/DESCRIPTION.md index bc1d5aa24..7879dc004 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -9,6 +9,11 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne # Release Notes +- v3.4.1(TBD) + + - Bumped vendored `urllib3` to 1.26.18 + - Bumped vendored `requests` to 2.31.0 + - v3.4.0(November 03,2023) - Added support for `use_logical_type` in `write_pandas`.