From af83e9dd9bf411735cdee1b27f6ea861980bb1d5 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 2 May 2023 15:15:14 +0200 Subject: [PATCH 1/7] Make compatible with requests 2.29.0. --- README.md | 2 +- meta/ee-requirements.txt | 2 +- plugins/doc_fragments/docker.py | 2 +- plugins/module_utils/_api/_import_helper.py | 10 ++++++++++ plugins/module_utils/_api/transport/npipeconn.py | 10 ++-------- plugins/module_utils/_api/transport/sshconn.py | 9 ++------- plugins/module_utils/_api/transport/unixconn.py | 4 ++-- tests/utils/constraints.txt | 1 - 8 files changed, 19 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 681c6ebb8..e82e0a8ed 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Some modules and plugins require Docker CLI, or other external, programs. Some r Installing the Docker SDK for Python also installs the requirements for the modules and plugins that use `requests`. If you want to directly install the Python libraries instead of the SDK, you need the following ones: -- [requests](https://pypi.org/project/requests/) (versions before 2.29.0); +- [requests](https://pypi.org/project/requests/); - [pywin32](https://pypi.org/project/pywin32/) when using named pipes on Windows with the Windows 32 API; - [paramiko](https://pypi.org/project/paramiko/) when using SSH to connect to the Docker daemon with `use_ssh_client=false`; - [pyOpenSSL](https://pypi.org/project/pyOpenSSL/) when using TLS to connect to the Docker daemon; diff --git a/meta/ee-requirements.txt b/meta/ee-requirements.txt index 565b1113e..ea3286bb3 100644 --- a/meta/ee-requirements.txt +++ b/meta/ee-requirements.txt @@ -4,7 +4,7 @@ docker urllib3 < 2.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611 -requests < 2.29 # TODO see https://github.com/ansible-collections/community.docker/issues/611 +requests paramiko # We assume that EEs are not based on Windows, and have Python >= 3.5. diff --git a/plugins/doc_fragments/docker.py b/plugins/doc_fragments/docker.py index 71ed5807f..4c537850e 100644 --- a/plugins/doc_fragments/docker.py +++ b/plugins/doc_fragments/docker.py @@ -289,7 +289,7 @@ class ModuleDocFragment(object): communicate with the Docker daemon. It uses code derived from the Docker SDK or Python that is included in this collection. requirements: - - requests < 2.29.0 (see U(https://github.com/ansible-collections/community.docker/issues/611)) + - requests - pywin32 (when using named pipes on Windows 32) - paramiko (when using SSH with I(use_ssh_client=false)) - pyOpenSSL (when using TLS) diff --git a/plugins/module_utils/_api/_import_helper.py b/plugins/module_utils/_api/_import_helper.py index f3ea504c9..0afe0b47c 100644 --- a/plugins/module_utils/_api/_import_helper.py +++ b/plugins/module_utils/_api/_import_helper.py @@ -42,15 +42,20 @@ class InvalidSchema(Exception): try: from requests.packages import urllib3 + from requests.packages.urllib3 import connection as urllib3_connection # pylint: disable=unused-import except ImportError: try: import urllib3 + from urllib3 import connection as urllib3_connection # pylint: disable=unused-import except ImportError: URLLIB3_IMPORT_ERROR = traceback.format_exc() class _HTTPConnectionPool(object): pass + class _HTTPConnection(object): + pass + class FakeURLLIB3(object): def __init__(self): self._collections = self @@ -63,7 +68,12 @@ def __init__(self): self.match_hostname = object() self.HTTPConnectionPool = _HTTPConnectionPool + class FakeURLLIB3Connection(object): + def __init__(self): + self.HTTPConnection = _HTTPConnection + urllib3 = FakeURLLIB3() + urllib3_connection = FakeURLLIB3Connection() # Monkey-patching match_hostname with a version that supports diff --git a/plugins/module_utils/_api/transport/npipeconn.py b/plugins/module_utils/_api/transport/npipeconn.py index 72a5c5899..912e465fe 100644 --- a/plugins/module_utils/_api/transport/npipeconn.py +++ b/plugins/module_utils/_api/transport/npipeconn.py @@ -10,24 +10,18 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.module_utils.six import PY3 from ansible.module_utils.six.moves.queue import Empty from .. import constants -from .._import_helper import HTTPAdapter, urllib3 +from .._import_helper import HTTPAdapter, urllib3, urllib3_connection from .basehttpadapter import BaseHTTPAdapter from .npipesocket import NpipeSocket -if PY3: - import http.client as httplib -else: - import httplib - RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class NpipeHTTPConnection(httplib.HTTPConnection, object): +class NpipeHTTPConnection(urllib3_connection.HTTPConnection, object): def __init__(self, npipe_path, timeout=60): super(NpipeHTTPConnection, self).__init__( 'localhost', timeout=timeout diff --git a/plugins/module_utils/_api/transport/sshconn.py b/plugins/module_utils/_api/transport/sshconn.py index 063c28826..a621d7557 100644 --- a/plugins/module_utils/_api/transport/sshconn.py +++ b/plugins/module_utils/_api/transport/sshconn.py @@ -24,12 +24,7 @@ from .basehttpadapter import BaseHTTPAdapter from .. import constants -if PY3: - import http.client as httplib -else: - import httplib - -from .._import_helper import HTTPAdapter, urllib3 +from .._import_helper import HTTPAdapter, urllib3, urllib3_connection PARAMIKO_IMPORT_ERROR = None try: @@ -120,7 +115,7 @@ def close(self): self.proc.terminate() -class SSHConnection(httplib.HTTPConnection, object): +class SSHConnection(urllib3_connection.HTTPConnection, object): def __init__(self, ssh_transport=None, timeout=60, host=None): super(SSHConnection, self).__init__( 'localhost', timeout=timeout diff --git a/plugins/module_utils/_api/transport/unixconn.py b/plugins/module_utils/_api/transport/unixconn.py index f46372f9a..1d15c5998 100644 --- a/plugins/module_utils/_api/transport/unixconn.py +++ b/plugins/module_utils/_api/transport/unixconn.py @@ -18,7 +18,7 @@ from .basehttpadapter import BaseHTTPAdapter from .. import constants -from .._import_helper import HTTPAdapter, urllib3 +from .._import_helper import HTTPAdapter, urllib3, urllib3_connection RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer @@ -35,7 +35,7 @@ def __init__(self, sock, *args, **kwargs): super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) -class UnixHTTPConnection(httplib.HTTPConnection, object): +class UnixHTTPConnection(urllib3_connection.HTTPConnection, object): def __init__(self, base_url, unix_socket, timeout=60): super(UnixHTTPConnection, self).__init__( diff --git a/tests/utils/constraints.txt b/tests/utils/constraints.txt index c8a2c39ed..a577641d8 100644 --- a/tests/utils/constraints.txt +++ b/tests/utils/constraints.txt @@ -16,7 +16,6 @@ paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for pyt paramiko < 3.0.0 ; python_version < '3.7' # paramiko 3.0.0 forces installation of a too new cryptography requests < 2.20.0 ; python_version < '2.7' # requests 2.20.0 drops support for python 2.6 requests < 2.28 ; python_version < '3.7' # requests 2.28.0 drops support for python < 3.7 -requests < 2.29 # TODO see https://github.com/ansible-collections/community.docker/issues/611 virtualenv < 16.0.0 ; python_version < '2.7' # virtualenv 16.0.0 and later require python 2.7 or later pyopenssl < 18.0.0 ; python_version < '2.7' # pyOpenSSL 18.0.0 and later require python 2.7 or later setuptools < 45 ; python_version <= '2.7' # setuptools 45 and later require python 3.5 or later From 1d2e482bf71af9de36f8bf6e3e9763a8e66712d8 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 2 May 2023 16:39:54 +0200 Subject: [PATCH 2/7] This fix should also work with urllib3 2.0 according to urllib3 maintainer. --- meta/ee-requirements.txt | 2 +- tests/utils/constraints.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/meta/ee-requirements.txt b/meta/ee-requirements.txt index ea3286bb3..de03ff0fd 100644 --- a/meta/ee-requirements.txt +++ b/meta/ee-requirements.txt @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later docker -urllib3 < 2.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611 +urllib3 requests paramiko diff --git a/tests/utils/constraints.txt b/tests/utils/constraints.txt index a577641d8..d8fcb61d5 100644 --- a/tests/utils/constraints.txt +++ b/tests/utils/constraints.txt @@ -10,7 +10,6 @@ coverage >= 4.5.4, < 5.0.0 ; python_version > '3.7' # coverage had a bug in < 4. cryptography >= 1.3.0, < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6 cryptography >= 1.3.0, < 3.4 ; python_version < '3.6' # cryptography 3.4 drops support for python 2.7 urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python 2.7 or later -urllib3 < 2.0.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for python 2.6 paramiko < 3.0.0 ; python_version < '3.7' # paramiko 3.0.0 forces installation of a too new cryptography From 48171e78bd73f5e040d89384af6e6bf25d0808c2 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 2 May 2023 16:41:15 +0200 Subject: [PATCH 3/7] Add changelog fragment. --- changelogs/fragments/613-requests.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/613-requests.yml diff --git a/changelogs/fragments/613-requests.yml b/changelogs/fragments/613-requests.yml new file mode 100644 index 000000000..356932edf --- /dev/null +++ b/changelogs/fragments/613-requests.yml @@ -0,0 +1,2 @@ +bugfixes: + - "Make vendored Docker SDK for Python code compatible with requests 2.29.0 and urllib3 2.0 (https://github.com/ansible-collections/community.docker/pull/613)." From 71e2f2208c4693d13ea9f8f2e9b6d0c721c8a169 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 2 May 2023 16:49:04 +0200 Subject: [PATCH 4/7] We still need the constraint for CI until Docker SDK for Python has a new release with a fix. --- tests/utils/constraints.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/constraints.txt b/tests/utils/constraints.txt index d8fcb61d5..a577641d8 100644 --- a/tests/utils/constraints.txt +++ b/tests/utils/constraints.txt @@ -10,6 +10,7 @@ coverage >= 4.5.4, < 5.0.0 ; python_version > '3.7' # coverage had a bug in < 4. cryptography >= 1.3.0, < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6 cryptography >= 1.3.0, < 3.4 ; python_version < '3.6' # cryptography 3.4 drops support for python 2.7 urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python 2.7 or later +urllib3 < 2.0.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for python 2.6 paramiko < 3.0.0 ; python_version < '3.7' # paramiko 3.0.0 forces installation of a too new cryptography From e5755bf882945900d996544be1613afc68a1bedd Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 3 May 2023 21:32:24 +0200 Subject: [PATCH 5/7] Make modifications to response_class as small as possible. --- .../module_utils/_api/transport/unixconn.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/plugins/module_utils/_api/transport/unixconn.py b/plugins/module_utils/_api/transport/unixconn.py index 1d15c5998..3b24fe46a 100644 --- a/plugins/module_utils/_api/transport/unixconn.py +++ b/plugins/module_utils/_api/transport/unixconn.py @@ -13,7 +13,6 @@ import socket from ansible.module_utils.six import PY2 -from ansible.module_utils.six.moves import http_client as httplib from .basehttpadapter import BaseHTTPAdapter from .. import constants @@ -24,17 +23,6 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class UnixHTTPResponse(httplib.HTTPResponse, object): - def __init__(self, sock, *args, **kwargs): - disable_buffering = kwargs.pop('disable_buffering', False) - if PY2: - # FIXME: We may need to disable buffering on Py3 as well, - # but there's no clear way to do it at the moment. See: - # https://github.com/docker/docker-py/issues/1799 - kwargs['buffering'] = not disable_buffering - super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) - - class UnixHTTPConnection(urllib3_connection.HTTPConnection, object): def __init__(self, base_url, unix_socket, timeout=60): @@ -58,10 +46,13 @@ def putheader(self, header, *values): self.disable_buffering = True def response_class(self, sock, *args, **kwargs): - if self.disable_buffering: - kwargs['disable_buffering'] = True + if PY2: + # FIXME: We may need to disable buffering on Py3 as well, + # but there's no clear way to do it at the moment. See: + # https://github.com/docker/docker-py/issues/1799 + kwargs['buffering'] = not self.disable_buffering - return UnixHTTPResponse(sock, *args, **kwargs) + return super(UnixHTTPConnection, self).response_class(sock, *args, **kwargs) class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): From 1be625317dabf520ab31f7ddfb5f02f3b097fd3d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 5 May 2023 21:20:49 +0200 Subject: [PATCH 6/7] Revert "We still need the constraint for CI until Docker SDK for Python has a new release with a fix." This reverts commit 698d544a1e08308e8bf8b4e56ab78c5079f9a17b. --- tests/utils/constraints.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/utils/constraints.txt b/tests/utils/constraints.txt index a577641d8..d8fcb61d5 100644 --- a/tests/utils/constraints.txt +++ b/tests/utils/constraints.txt @@ -10,7 +10,6 @@ coverage >= 4.5.4, < 5.0.0 ; python_version > '3.7' # coverage had a bug in < 4. cryptography >= 1.3.0, < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6 cryptography >= 1.3.0, < 3.4 ; python_version < '3.6' # cryptography 3.4 drops support for python 2.7 urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python 2.7 or later -urllib3 < 2.0.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for python 2.6 paramiko < 3.0.0 ; python_version < '3.7' # paramiko 3.0.0 forces installation of a too new cryptography From 9799cc891d01b2c67793d5f2ad65d8730103fc29 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 5 May 2023 21:28:17 +0200 Subject: [PATCH 7/7] The pip coming with the ansible-core 2.11 alpine3 image seems to be too old. --- .github/workflows/ansible-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index e2a5e30ac..2aa1aed0b 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -101,6 +101,10 @@ jobs: - '' target: - '' + extra-constraints: + # Specifying this other than '' likely destroys change detection, but at least it will make + # CI pass when necessary... + - '' exclude: - ansible: '' include: @@ -117,6 +121,7 @@ jobs: docker: alpine3 python: '' target: azp/4/ + extra-constraints: urllib3 < 2.0.0 - ansible: '2.11' docker: alpine3 python: '' @@ -144,6 +149,9 @@ jobs: git clone --depth=1 --single-branch https://github.com/ansible-collections/community.crypto.git ../../community/crypto ; git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git ../../community/general + ${{ matrix.extra-constraints && format('; echo ''{0}'' >> tests/utils/constraints.txt', matrix.extra-constraints) || '' }} + ; + cat tests/utils/constraints.txt pull-request-change-detection: 'true' target: ${{ matrix.target }} target-python-version: ${{ matrix.python }}