Pull in code from requests-unixsocket and support urllib3 v2 (#29)

.mypy.ini

@@ -24,6 +24,11 @@ warn_unused_ignores = True
 # See: https://github.com/python/mypy/issues/8754#issuecomment-622376701
 implicit_reexport = True
 
+# The code forked from request-unixsocket is not compliant
+[mypy-vplan.unixsocket.*]
+check_untyped_defs = False
+allow_untyped_defs = True
+
 # It's hard to make tests compliant using unittest.mock
 [mypy-tests.*]
 check_untyped_defs = False
@@ -33,10 +38,6 @@ allow_untyped_defs = True
 [mypy-pytest]
 ignore_missing_imports = True
 
-# There is no type hinting for requests_unixsocket
-[mypy-requests_unixsocket]
-ignore_missing_imports = True
-
 # There is no type hinting for apscheduler
 [mypy-apscheduler.*]
 ignore_missing_imports = True

.run/tasks/server.sh

@@ -29,6 +29,8 @@ task_server() {
       run_task rmdb
    fi
 
+   chmod 700 config/local/vplan/server/db
+
    poetry_run uvicorn vplan.engine.server:API \
       --port 8080 \
       --app-dir src --reload \

Changelog

@@ -1,6 +1,10 @@
+Version 0.7.0     unreleased
+
+	* Pull in code from requests-unixsocket and patch to support new urllib3.
+
 Version 0.6.5     10 Jun 2023
 
-	* Fix functional problems by pinning urllib<2.
+	* Fix functional problems by pinning urllib3<2.
 
 Version 0.6.4     08 Jun 2023
 

DEVELOPER.md

@@ -8,7 +8,7 @@ This code runs as a daemon and is intended for use on Linux and UNIX-like platfo
 
 The systemd design is heavily based on the excellent [python-systemd-tutorial](https://github.com/torfsen/python-systemd-tutorial).  See also the [specifiers](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Specifiers) documentation (for constructs like `%h`).
 
-Uvicorn is using a user-private UNIX socket rather than opening a port like 8080.  For the socket setup, I followed notes [here](https://gist.github.com/kylemanna/d193aaa6b33a89f649524ad27ce47c4b) and [here](https://stackoverflow.com/questions/52507089/running-uvicorn-with-unix-socket).  I'm using [requests-unixsocket](https://pypi.org/project/requests-unixsocket/) to make requests to the UNIX socket.
+Uvicorn is using a user-private UNIX socket rather than opening a port like 8080.  For the socket setup, I followed notes [here](https://gist.github.com/kylemanna/d193aaa6b33a89f649524ad27ce47c4b) and [here](https://stackoverflow.com/questions/52507089/running-uvicorn-with-unix-socket).
 
 ## Packaging and Dependencies
 

NOTICE

@@ -14,3 +14,14 @@ under the following license:
 
 The contents of the Apache License can be found in the LICENSE file, or 
 can be downloaded from http://www.apache.org/licenses/LICENSE-2.0 .
+
+The source code under src/vplan/unixsocket and tests/unixsocket originated at
+msabramo/requests-unixsocket on GitHub.  The code was pulled from the v0.3.0
+release on PyPI because there is no v0.3.0 tag at GitHub:
+
+   https://github.com/msabramo/requests-unixsocket
+   https://pypi.org/project/requests-unixsocket/0.3.0/#files
+
+The requests-unixsocket repository contains a LICENSE file for the Apache
+License, Version 2.0, but there is no copyright statement or other explicit
+attribution.  The PyPI entry and shows the author as <marc@marc-abramowitz.com>.

pyproject.toml

@@ -44,9 +44,7 @@ uvicorn = "~0, >=0.20.0"
 pydantic-yaml = "~0, >=0.11.2"
 semver = "^3.0.0"
 click = "^8.1.3"
-urllib3 = "<2" # compatibility problems with v2.x
 requests = "^2.31.0"
-requests-unixsocket = "~0, >=0.3.0"
 SQLAlchemy = "^1.4.46"
 APScheduler = "^3.10.0"
 python-dotenv = "^1.0.0"
@@ -73,6 +71,8 @@ colorama = "~0, >=0.4.5"
 httpx = "~0, >=0.23.3"
 responses = "~0, >=0.22.0"
 types-pyyaml = "^6.0.12.9"
+waitress = "^2.1.2"
+types-waitress = "^2.1.4.8"
 
 [tool.black]
 line-length = 132

src/vplan/client/client.py

@@ -10,14 +10,14 @@
 
 import click
 import requests
-import requests_unixsocket
 from requests import HTTPError, Response
 
+import vplan.unixsocket as requests_unixsocket
 from vplan.client.config import api_url
 from vplan.interface import Account, PlanSchema, Status, Version
 
 # Add support in requests for http+unix:// URLs to use a UNIX socket
-requests_unixsocket.monkeypatch()
+requests_unixsocket.monkeypatch()  # type: ignore
 
 
 def _url(endpoint: str) -> str:

src/vplan/unixsocket/README.md

@@ -0,0 +1,5 @@
+## Forked Code
+
+This code was forked from requests-unixsocket, with source taken from [v0.3.0 on PyPI](https://pypi.org/project/requests-unixsocket/0.3.0/#files).  The original source code is on GitHub at [msabramo/requests-unixsocket](https://github.com/msabramo/requests-unixsocket), but v0.3.0 doesn't appear in the tags there.  This code is licensed under Apache v2, so this is permitted use.
+
+I forked the code because it's incompatible with [urllib3 v2](https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html), which requests moved to as of [v2.30.0](https://github.com/psf/requests/releases/tag/v2.30.0).  We need to be on requests [>= v2.31.0](https://github.com/psf/requests/releases/tag/v2.31.0) due to [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681).  The problem with requests-unixsocket is tracked in [issue #70](https://github.com/msabramo/requests-unixsocket/issues/70) and fixed in [PR #69](https://github.com/msabramo/requests-unixsocket/pull/69).  However, as of this writing, the requests-unixsocket maintainer hasn't responded to either the issue or the PR.  Given how small the code is, it seems safer and simpler to just pull it in rather than waiting for a new package to be released on PyPI.

src/vplan/unixsocket/__init__.py

@@ -0,0 +1,80 @@
+# pylint: disable=super-with-arguments,keyword-arg-before-vararg,invalid-name,redefined-outer-name,unused-argument:
+
+# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details
+
+import sys
+
+import requests
+
+from .adapters import UnixAdapter
+
+DEFAULT_SCHEME = "http+unix://"
+
+
+class Session(requests.Session):
+    def __init__(self, url_scheme=DEFAULT_SCHEME, *args, **kwargs):
+        super(Session, self).__init__(*args, **kwargs)
+        self.mount(url_scheme, UnixAdapter())
+
+
+class monkeypatch:
+    def __init__(self, url_scheme=DEFAULT_SCHEME):
+        self.session = Session()
+        requests = self._get_global_requests_module()
+
+        # Methods to replace
+        self.methods = ("request", "get", "head", "post", "patch", "put", "delete", "options")
+        # Store the original methods
+        self.orig_methods = dict((m, requests.__dict__[m]) for m in self.methods)
+        # Monkey patch
+        g = globals()
+        for m in self.methods:
+            requests.__dict__[m] = g[m]
+
+    def _get_global_requests_module(self):
+        return sys.modules["requests"]
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        requests = self._get_global_requests_module()
+        for m in self.methods:
+            requests.__dict__[m] = self.orig_methods[m]
+
+
+# These are the same methods defined for the global requests object
+def request(method, url, **kwargs):
+    session = Session()
+    return session.request(method=method, url=url, **kwargs)
+
+
+def get(url, **kwargs):
+    kwargs.setdefault("allow_redirects", True)
+    return request("get", url, **kwargs)
+
+
+def head(url, **kwargs):
+    kwargs.setdefault("allow_redirects", False)
+    return request("head", url, **kwargs)
+
+
+def post(url, data=None, json=None, **kwargs):
+    return request("post", url, data=data, json=json, **kwargs)
+
+
+def patch(url, data=None, **kwargs):
+    return request("patch", url, data=data, **kwargs)
+
+
+def put(url, data=None, **kwargs):
+    return request("put", url, data=data, **kwargs)
+
+
+def delete(url, **kwargs):
+    return request("delete", url, **kwargs)
+
+
+def options(url, **kwargs):
+    kwargs.setdefault("allow_redirects", True)
+    return request("options", url, **kwargs)

src/vplan/unixsocket/adapters.py

@@ -0,0 +1,76 @@
+# pylint: disable=super-with-arguments,keyword-arg-before-vararg,ungrouped-imports:
+
+# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details
+
+import socket
+
+import urllib3
+from requests.adapters import HTTPAdapter
+from requests.compat import unquote, urlparse
+
+
+# The following was adapted from some code from docker-py
+# https://github.com/docker/docker-py/blob/master/docker/transport/unixconn.py
+class UnixHTTPConnection(urllib3.connection.HTTPConnection):
+    def __init__(self, unix_socket_url, timeout=60):
+        """Create an HTTP connection to a unix domain socket
+
+        :param unix_socket_url: A URL with a scheme of 'http+unix' and the
+        netloc is a percent-encoded path to a unix domain socket. E.g.:
+        'http+unix://%2Ftmp%2Fprofilesvc.sock/status/pid'
+        """
+        super(UnixHTTPConnection, self).__init__("localhost", timeout=timeout)
+        self.unix_socket_url = unix_socket_url
+        self.timeout = timeout
+        self.sock = None
+
+    def __del__(self):  # base class does not have d'tor
+        if self.sock:
+            self.sock.close()
+
+    def connect(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.settimeout(self.timeout)
+        socket_path = unquote(urlparse(self.unix_socket_url).netloc)
+        sock.connect(socket_path)
+        self.sock = sock
+
+
+class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
+    def __init__(self, socket_path, timeout=60):
+        super(UnixHTTPConnectionPool, self).__init__("localhost", timeout=timeout)
+        self.socket_path = socket_path
+        self.timeout = timeout
+
+    def _new_conn(self):
+        return UnixHTTPConnection(self.socket_path, self.timeout)
+
+
+class UnixAdapter(HTTPAdapter):
+    def __init__(self, timeout=60, pool_connections=25, *args, **kwargs):
+        super(UnixAdapter, self).__init__(*args, **kwargs)
+        self.timeout = timeout
+        self.pools = urllib3._collections.RecentlyUsedContainer(pool_connections, dispose_func=lambda p: p.close())
+
+    def get_connection(self, url, proxies=None):
+        proxies = proxies or {}
+        proxy = proxies.get(urlparse(url.lower()).scheme)
+
+        if proxy:
+            raise ValueError("%s does not support specifying proxies" % self.__class__.__name__)
+
+        with self.pools.lock:
+            pool = self.pools.get(url)
+            if pool:
+                return pool
+
+            pool = UnixHTTPConnectionPool(url, self.timeout)
+            self.pools[url] = pool
+
+        return pool
+
+    def request_url(self, request, proxies):
+        return request.path_url
+
+    def close(self):
+        self.pools.clear()

tests/unixsocket/README.md

@@ -0,0 +1,5 @@
+## Forked Code
+
+This code was forked from requests-unixsocket, with source taken from [v0.3.0 on PyPI](https://pypi.org/project/requests-unixsocket/0.3.0/#files).  The original source code is on GitHub at [msabramo/requests-unixsocket](https://github.com/msabramo/requests-unixsocket), but v0.3.0 doesn't appear in the tags there.  This code is licensed under Apache v2, so this is permitted use.
+
+I forked the code because it's incompatible with [urllib3 v2](https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html), which requests moved to as of [v2.30.0](https://github.com/psf/requests/releases/tag/v2.30.0).  We need to be on requests [>= v2.31.0](https://github.com/psf/requests/releases/tag/v2.31.0) due to [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681).  The problem with requests-unixsocket is tracked in [issue #70](https://github.com/msabramo/requests-unixsocket/issues/70) and fixed in [PR #69](https://github.com/msabramo/requests-unixsocket/pull/69).  However, as of this writing, the requests-unixsocket maintainer hasn't responded to either the issue or the PR.  Given how small the code is, it seems safer and simpler to just pull it in rather than waiting for a new package to be released on PyPI.

tests/unixsocket/__init__.py

@@ -0,0 +1 @@
+__all__ = []  # type: ignore