diff --git a/docs/conf.py b/docs/conf.py index e1730a72..7a7d306a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,7 +47,7 @@ autosectionlabel_maxdepth = 2 # AutodocTypehints settings. -autodoc_typehints = 'description' +autodoc_typehints = "description" always_document_param_types = True typehints_defaults = "comma" diff --git a/docs/examples/opentelemetry/main.py b/docs/examples/opentelemetry/main.py index 716c35e1..91c9268d 100755 --- a/docs/examples/opentelemetry/main.py +++ b/docs/examples/opentelemetry/main.py @@ -2,8 +2,8 @@ import time -import valkey import uptrace +import valkey from opentelemetry import trace from opentelemetry.instrumentation.valkey import ValkeyInstrumentor diff --git a/tests/conftest.py b/tests/conftest.py index d4aa2181..09ac315b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,8 +9,9 @@ import valkey from packaging.version import Version from valkey import Sentinel +from valkey._parsers import parse_url from valkey.backoff import NoBackoff -from valkey.connection import Connection, parse_url +from valkey.connection import Connection from valkey.exceptions import ValkeyClusterException from valkey.retry import Retry @@ -275,7 +276,7 @@ def _get_client( cluster_mode = VALKEY_INFO["cluster_enabled"] if not cluster_mode: - url_options = parse_url(valkey_url) + url_options = parse_url(valkey_url, False) url_options.update(kwargs) pool = valkey.ConnectionPool(**url_options) client = cls(connection_pool=pool) diff --git a/tests/test_asyncio/conftest.py b/tests/test_asyncio/conftest.py index c84fe79d..e4f2f7ba 100644 --- a/tests/test_asyncio/conftest.py +++ b/tests/test_asyncio/conftest.py @@ -5,9 +5,10 @@ import pytest_asyncio import valkey.asyncio as valkey from tests.conftest import VALKEY_INFO +from valkey._parsers import parse_url from valkey.asyncio import Sentinel from valkey.asyncio.client import Monitor -from valkey.asyncio.connection import Connection, parse_url +from valkey.asyncio.connection import Connection from valkey.asyncio.retry import Retry from valkey.backoff import NoBackoff @@ -54,7 +55,7 @@ async def client_factory( cluster_mode = VALKEY_INFO["cluster_enabled"] if not cluster_mode: single = kwargs.pop("single_connection_client", False) or single_connection - url_options = parse_url(url) + url_options = parse_url(url, True) url_options.update(kwargs) pool = valkey.ConnectionPool(**url_options) client = cls(connection_pool=pool) @@ -269,4 +270,4 @@ def valkey_url(request): @pytest.fixture() def connect_args(request): url = request.config.getoption("--valkey-url") - return parse_url(url) + return parse_url(url, True) diff --git a/tests/test_asyncio/test_connection.py b/tests/test_asyncio/test_connection.py index 4da68b43..6851cfda 100644 --- a/tests/test_asyncio/test_connection.py +++ b/tests/test_asyncio/test_connection.py @@ -11,9 +11,10 @@ _AsyncRESP2Parser, _AsyncRESP3Parser, _AsyncRESPBase, + parse_url, ) from valkey.asyncio import ConnectionPool, Valkey -from valkey.asyncio.connection import Connection, UnixDomainSocketConnection, parse_url +from valkey.asyncio.connection import Connection, UnixDomainSocketConnection from valkey.asyncio.retry import Retry from valkey.backoff import NoBackoff from valkey.exceptions import ConnectionError, InvalidResponse, TimeoutError @@ -300,7 +301,7 @@ async def test_pool_auto_close(request, from_url): """Verify that basic Valkey instances have auto_close_connection_pool set to True""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) async def get_valkey_connection(): if from_url: @@ -342,7 +343,7 @@ async def test_pool_auto_close_disable(request): """Verify that auto_close_connection_pool can be disabled (deprecated)""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) async def get_valkey_connection(): url_args["auto_close_connection_pool"] = False @@ -361,7 +362,7 @@ async def test_valkey_connection_pool(request, from_url): have auto_close_connection_pool set to False""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) pool = None @@ -393,7 +394,7 @@ async def test_valkey_from_pool(request, from_url): have auto_close_connection_pool set to True""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) pool = None diff --git a/tests/test_asyncio/test_connection_pool.py b/tests/test_asyncio/test_connection_pool.py index ce8d792a..2e0d21aa 100644 --- a/tests/test_asyncio/test_connection_pool.py +++ b/tests/test_asyncio/test_connection_pool.py @@ -5,7 +5,8 @@ import pytest_asyncio import valkey.asyncio as valkey from tests.conftest import skip_if_server_version_lt -from valkey.asyncio.connection import Connection, to_bool +from valkey._parsers.url_parser import to_bool +from valkey.asyncio.connection import Connection from .compat import aclosing, mock from .conftest import asynccontextmanager diff --git a/tests/test_connection.py b/tests/test_connection.py index 9c60aa8f..13617e8f 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -6,14 +6,9 @@ import pytest import valkey from valkey import ConnectionPool, Valkey -from valkey._parsers import _HiredisParser, _RESP2Parser, _RESP3Parser +from valkey._parsers import _HiredisParser, _RESP2Parser, _RESP3Parser, parse_url from valkey.backoff import NoBackoff -from valkey.connection import ( - Connection, - SSLConnection, - UnixDomainSocketConnection, - parse_url, -) +from valkey.connection import Connection, SSLConnection, UnixDomainSocketConnection from valkey.exceptions import ConnectionError, InvalidResponse, TimeoutError from valkey.retry import Retry from valkey.utils import HIREDIS_AVAILABLE @@ -222,7 +217,7 @@ def test_pool_auto_close(request, from_url): """Verify that basic Valkey instances have auto_close_connection_pool set to True""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, False) def get_valkey_connection(): if from_url: @@ -240,7 +235,7 @@ def test_valkey_connection_pool(request, from_url): have auto_close_connection_pool set to False""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) pool = None @@ -272,7 +267,7 @@ def test_valkey_from_pool(request, from_url): have auto_close_connection_pool set to True""" url: str = request.config.getoption("--valkey-url") - url_args = parse_url(url) + url_args = parse_url(url, True) pool = None diff --git a/tests/test_connection_pool.py b/tests/test_connection_pool.py index 07806fa7..9950ec01 100644 --- a/tests/test_connection_pool.py +++ b/tests/test_connection_pool.py @@ -7,7 +7,7 @@ import pytest import valkey -from valkey.connection import to_bool +from valkey._parsers.url_parser import to_bool from valkey.utils import SSL_AVAILABLE from .conftest import _get_client, skip_if_server_version_lt diff --git a/valkey/_parsers/__init__.py b/valkey/_parsers/__init__.py index 6cc32e3c..b1976013 100644 --- a/valkey/_parsers/__init__.py +++ b/valkey/_parsers/__init__.py @@ -4,6 +4,7 @@ from .hiredis import _AsyncHiredisParser, _HiredisParser from .resp2 import _AsyncRESP2Parser, _RESP2Parser from .resp3 import _AsyncRESP3Parser, _RESP3Parser +from .url_parser import parse_url __all__ = [ "AsyncCommandsParser", @@ -17,4 +18,5 @@ "_HiredisParser", "_RESP2Parser", "_RESP3Parser", + "parse_url", ] diff --git a/valkey/_parsers/url_parser.py b/valkey/_parsers/url_parser.py new file mode 100644 index 00000000..6d93f734 --- /dev/null +++ b/valkey/_parsers/url_parser.py @@ -0,0 +1,104 @@ +from types import MappingProxyType +from typing import Callable, Mapping, Optional +from urllib.parse import ParseResult, parse_qs, unquote, urlparse + +from valkey.asyncio.connection import ConnectKwargs +from valkey.asyncio.connection import SSLConnection as SSLConnectionAsync +from valkey.asyncio.connection import ( + UnixDomainSocketConnection as UnixDomainSocketConnectionAsync, +) +from valkey.connection import SSLConnection, UnixDomainSocketConnection + + +def to_bool(value) -> Optional[bool]: + if value is None or value == "": + return None + if isinstance(value, str) and value.upper() in FALSE_STRINGS: + return False + return bool(value) + + +FALSE_STRINGS = ("0", "F", "FALSE", "N", "NO") + +URL_QUERY_ARGUMENT_PARSERS: Mapping[str, Callable[..., object]] = MappingProxyType( + { + "db": int, + "socket_timeout": float, + "socket_connect_timeout": float, + "socket_keepalive": to_bool, + "retry_on_timeout": to_bool, + "max_connections": int, + "health_check_interval": int, + "ssl_check_hostname": to_bool, + "timeout": float, + } +) + + +def parse_url(url: str, async_connection: bool): + + if not ( + url.startswith("valkey://") + or url.startswith("valkeys://") + or url.startswith("unix://") + ): + raise ValueError( + "Valkey URL must specify one of the following " + "schemes (valkey://, valkeys://, unix://)" + ) + + parsed: ParseResult = urlparse(url) + kwargs: ConnectKwargs = {} + + for name, value_list in parse_qs(parsed.query).items(): + if value_list and len(value_list) > 0: + value = unquote(value_list[0]) + parser = URL_QUERY_ARGUMENT_PARSERS.get(name) + if parser: + try: + kwargs[name] = parser(value) + except (TypeError, ValueError): + raise ValueError(f"Invalid value for `{name}` in connection URL.") + else: + kwargs[name] = value + + if parsed.username: + kwargs["username"] = unquote(parsed.username) + if parsed.password: + kwargs["password"] = unquote(parsed.password) + + # We only support valkey://, valkeys:// and unix:// schemes. + if parsed.scheme == "unix": + if parsed.path: + kwargs["path"] = unquote(parsed.path) + kwargs["connection_class"] = ( + UnixDomainSocketConnectionAsync + if async_connection + else UnixDomainSocketConnection + ) + + elif parsed.scheme in ("valkey", "valkeys"): + if parsed.hostname: + kwargs["host"] = unquote(parsed.hostname) + if parsed.port: + kwargs["port"] = int(parsed.port) + + # If there's a path argument, use it as the db argument if a + # querystring value wasn't specified + if parsed.path and "db" not in kwargs: + try: + kwargs["db"] = int(unquote(parsed.path).replace("/", "")) + except (AttributeError, ValueError): + pass + + if parsed.scheme == "valkeys": + kwargs["connection_class"] = ( + SSLConnectionAsync if async_connection else SSLConnection + ) + else: + valid_schemes = "valkey://, valkeys://, unix://" + raise ValueError( + f"Valkey URL must specify one of the following schemes ({valid_schemes})" + ) + + return kwargs diff --git a/valkey/asyncio/cluster.py b/valkey/asyncio/cluster.py index b240b4df..56de44f7 100644 --- a/valkey/asyncio/cluster.py +++ b/valkey/asyncio/cluster.py @@ -25,19 +25,14 @@ DEFAULT_EVICTION_POLICY, AbstractCache, ) -from valkey._parsers import AsyncCommandsParser, Encoder +from valkey._parsers import AsyncCommandsParser, Encoder, parse_url from valkey._parsers.helpers import ( _ValkeyCallbacks, _ValkeyCallbacksRESP2, _ValkeyCallbacksRESP3, ) from valkey.asyncio.client import ResponseCallbackT -from valkey.asyncio.connection import ( - Connection, - DefaultParser, - SSLConnection, - parse_url, -) +from valkey.asyncio.connection import Connection, DefaultParser, SSLConnection from valkey.asyncio.lock import Lock from valkey.asyncio.retry import Retry from valkey.backoff import default_backoff @@ -211,7 +206,7 @@ def from_url(cls, url: str, **kwargs: Any) -> "ValkeyCluster": :class:`~valkey.asyncio.connection.Connection` when created. In the case of conflicting arguments, querystring arguments are used. """ - kwargs.update(parse_url(url)) + kwargs.update(parse_url(url, True)) if kwargs.pop("connection_class", None) is SSLConnection: kwargs["ssl"] = True return cls(**kwargs) diff --git a/valkey/asyncio/connection.py b/valkey/asyncio/connection.py index 77b329a8..573d41b3 100644 --- a/valkey/asyncio/connection.py +++ b/valkey/asyncio/connection.py @@ -9,10 +9,8 @@ import weakref from abc import abstractmethod from itertools import chain -from types import MappingProxyType from typing import ( Any, - Callable, Iterable, List, Mapping, @@ -25,7 +23,6 @@ TypeVar, Union, ) -from urllib.parse import ParseResult, parse_qs, unquote, urlparse # the functionality is available in 3.11.x but has a major issue before # 3.11.3. See https://github.com/redis/redis-py/issues/2633 @@ -986,32 +983,6 @@ def _error_message(self, exception: BaseException) -> str: ) -FALSE_STRINGS = ("0", "F", "FALSE", "N", "NO") - - -def to_bool(value) -> Optional[bool]: - if value is None or value == "": - return None - if isinstance(value, str) and value.upper() in FALSE_STRINGS: - return False - return bool(value) - - -URL_QUERY_ARGUMENT_PARSERS: Mapping[str, Callable[..., object]] = MappingProxyType( - { - "db": int, - "socket_timeout": float, - "socket_connect_timeout": float, - "socket_keepalive": to_bool, - "retry_on_timeout": to_bool, - "max_connections": int, - "health_check_interval": int, - "ssl_check_hostname": to_bool, - "timeout": float, - } -) - - class ConnectKwargs(TypedDict, total=False): username: str password: str @@ -1022,58 +993,6 @@ class ConnectKwargs(TypedDict, total=False): path: str -def parse_url(url: str) -> ConnectKwargs: - parsed: ParseResult = urlparse(url) - kwargs: ConnectKwargs = {} - - for name, value_list in parse_qs(parsed.query).items(): - if value_list and len(value_list) > 0: - value = unquote(value_list[0]) - parser = URL_QUERY_ARGUMENT_PARSERS.get(name) - if parser: - try: - kwargs[name] = parser(value) - except (TypeError, ValueError): - raise ValueError(f"Invalid value for `{name}` in connection URL.") - else: - kwargs[name] = value - - if parsed.username: - kwargs["username"] = unquote(parsed.username) - if parsed.password: - kwargs["password"] = unquote(parsed.password) - - # We only support valkey://, valkeys:// and unix:// schemes. - if parsed.scheme == "unix": - if parsed.path: - kwargs["path"] = unquote(parsed.path) - kwargs["connection_class"] = UnixDomainSocketConnection - - elif parsed.scheme in ("valkey", "valkeys"): - if parsed.hostname: - kwargs["host"] = unquote(parsed.hostname) - if parsed.port: - kwargs["port"] = int(parsed.port) - - # If there's a path argument, use it as the db argument if a - # querystring value wasn't specified - if parsed.path and "db" not in kwargs: - try: - kwargs["db"] = int(unquote(parsed.path).replace("/", "")) - except (AttributeError, ValueError): - pass - - if parsed.scheme == "valkeys": - kwargs["connection_class"] = SSLConnection - else: - valid_schemes = "valkey://, valkeys://, unix://" - raise ValueError( - f"Valkey URL must specify one of the following schemes ({valid_schemes})" - ) - - return kwargs - - _CP = TypeVar("_CP", bound="ConnectionPool") @@ -1132,7 +1051,9 @@ def from_url(cls: Type[_CP], url: str, **kwargs) -> _CP: class initializer. In the case of conflicting arguments, querystring arguments always win. """ - url_options = parse_url(url) + from .._parsers.url_parser import parse_url + + url_options = parse_url(url, True) kwargs.update(url_options) return cls(**kwargs) diff --git a/valkey/cluster.py b/valkey/cluster.py index 0a4a3504..b9c125a8 100644 --- a/valkey/cluster.py +++ b/valkey/cluster.py @@ -6,13 +6,13 @@ from collections import OrderedDict from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from valkey._parsers import CommandsParser, Encoder +from valkey._parsers import CommandsParser, Encoder, parse_url from valkey._parsers.helpers import parse_scan from valkey.backoff import default_backoff from valkey.client import CaseInsensitiveDict, PubSub, Valkey from valkey.commands import READ_COMMANDS, ValkeyClusterCommands from valkey.commands.helpers import list_or_args -from valkey.connection import ConnectionPool, DefaultParser, parse_url +from valkey.connection import ConnectionPool, DefaultParser from valkey.crc import VALKEY_CLUSTER_HASH_SLOTS, key_slot from valkey.exceptions import ( AskError, @@ -580,7 +580,7 @@ def __init__( from_url = False if url is not None: from_url = True - url_options = parse_url(url) + url_options = parse_url(url, False) if "path" in url_options: raise ValkeyClusterException( "ValkeyCluster does not currently support Unix Domain " diff --git a/valkey/connection.py b/valkey/connection.py index 29d3fbb0..d34588d5 100644 --- a/valkey/connection.py +++ b/valkey/connection.py @@ -10,7 +10,6 @@ from queue import Empty, Full, LifoQueue from time import time from typing import Any, Callable, List, Optional, Sequence, Type, Union -from urllib.parse import parse_qs, unquote, urlparse from ._cache import ( DEFAULT_ALLOW_LIST, @@ -948,85 +947,6 @@ def _error_message(self, exception): FALSE_STRINGS = ("0", "F", "FALSE", "N", "NO") -def to_bool(value): - if value is None or value == "": - return None - if isinstance(value, str) and value.upper() in FALSE_STRINGS: - return False - return bool(value) - - -URL_QUERY_ARGUMENT_PARSERS = { - "db": int, - "socket_timeout": float, - "socket_connect_timeout": float, - "socket_keepalive": to_bool, - "retry_on_timeout": to_bool, - "retry_on_error": list, - "max_connections": int, - "health_check_interval": int, - "ssl_check_hostname": to_bool, - "timeout": float, -} - - -def parse_url(url): - if not ( - url.startswith("valkey://") - or url.startswith("valkeys://") - or url.startswith("unix://") - ): - raise ValueError( - "Valkey URL must specify one of the following " - "schemes (valkey://, valkeys://, unix://)" - ) - - url = urlparse(url) - kwargs = {} - - for name, value in parse_qs(url.query).items(): - if value and len(value) > 0: - value = unquote(value[0]) - parser = URL_QUERY_ARGUMENT_PARSERS.get(name) - if parser: - try: - kwargs[name] = parser(value) - except (TypeError, ValueError): - raise ValueError(f"Invalid value for `{name}` in connection URL.") - else: - kwargs[name] = value - - if url.username: - kwargs["username"] = unquote(url.username) - if url.password: - kwargs["password"] = unquote(url.password) - - # We only support valkey://, valkeys:// and unix:// schemes. - if url.scheme == "unix": - if url.path: - kwargs["path"] = unquote(url.path) - kwargs["connection_class"] = UnixDomainSocketConnection - - else: # implied: url.scheme in ("valkey", "valkeys"): - if url.hostname: - kwargs["host"] = unquote(url.hostname) - if url.port: - kwargs["port"] = int(url.port) - - # If there's a path argument, use it as the db argument if a - # querystring value wasn't specified - if url.path and "db" not in kwargs: - try: - kwargs["db"] = int(unquote(url.path).replace("/", "")) - except (AttributeError, ValueError): - pass - - if url.scheme == "valkeys": - kwargs["connection_class"] = SSLConnection - - return kwargs - - class ConnectionPool: """ Create a connection pool. ``If max_connections`` is set, then this @@ -1080,8 +1000,9 @@ def from_url(cls, url, **kwargs): class initializer. In the case of conflicting arguments, querystring arguments always win. """ - url_options = parse_url(url) + from ._parsers.url_parser import parse_url + url_options = parse_url(url, False) if "connection_class" in kwargs: url_options["connection_class"] = kwargs["connection_class"]