Refactor store creation from URL

.gitignore

@@ -110,5 +110,10 @@ Pipfile.lock
 
 # minimalkv
 store/
+old/
+
 # Exploratory code
-exp.py
+exploration_scripts/
+
+# terraform
+terraform/.terraform*

docs/changes.rst

@@ -1,6 +1,14 @@
 Changelog
 *********
 
+1.7.0
+=====
+* Deprecated ``get_store``, ``url2dict``, ``_parse_userinfo``, and ``extract_params``.
+
+  * ``get_store_from_url`` should be used to create stores from a URL
+
+* Added ``from_url`` and ``from_parsed_url`` to each store.
+
 1.6.0
 =====
 

environment.yml

@@ -6,6 +6,7 @@ dependencies:
   - azure-storage-blob
   - boto
   - boto3
+  - mypy-boto3-s3
   - docker-compose
   - dulwich
   - fsspec

minimalkv/__init__.py

@@ -8,9 +8,9 @@
 from minimalkv._get_store import get_store, get_store_from_url
 from minimalkv._key_value_store import KeyValueStore, UrlKeyValueStore
 from minimalkv._mixins import CopyMixin, TimeToLiveMixin, UrlMixin
-from minimalkv._store_creation import create_store
+from minimalkv._old_store_creation import create_store
+from minimalkv._old_urls import url2dict
 from minimalkv._store_decoration import decorate_store
-from minimalkv._urls import url2dict
 
 try:
     import pkg_resources

minimalkv/_boto.py

@@ -10,13 +10,23 @@ def _get_s3bucket(
     from boto.s3.connection import S3ResponseError  # type: ignore
     from boto.s3.connection import OrdinaryCallingFormat, S3Connection
 
-    s3con = S3Connection(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        host=host,
-        is_secure=False,
-        calling_format=OrdinaryCallingFormat(),
-    )
+    s3_connection_params = {
+        "aws_access_key_id": access_key,
+        "aws_secret_access_key": secret_key,
+        "is_secure": False,
+        "calling_format": OrdinaryCallingFormat(),
+    }
+
+    # Split up the host into host and port.
+    if ":" in host:
+        host, port = host.split(":")
+        s3_connection_params["host"] = host
+        s3_connection_params["port"] = int(port)
+    else:
+        s3_connection_params["host"] = host
+
+    s3con = S3Connection(**s3_connection_params)
+
     # add access key prefix to bucket name, unless explicitly prohibited
     if force_bucket_suffix and not bucket.lower().endswith("-" + access_key.lower()):
         bucket = bucket + "-" + access_key.lower()

minimalkv/_get_store.py

@@ -1,18 +1,25 @@
 from functools import reduce
-from typing import Any
+from typing import Any, Dict, List, Optional, Type
+from warnings import warn
+
+from uritools import SplitResult, urisplit
 
 from minimalkv._key_value_store import KeyValueStore
-from minimalkv._urls import url2dict
 
 
-def get_store_from_url(url: str) -> KeyValueStore:
+def get_store_from_url(
+    url: str, store_cls: Optional[Type[KeyValueStore]] = None
+) -> KeyValueStore:
     """
     Take a URL and return a minimalkv store according to the parameters in the URL.
 
     Parameters
     ----------
     url : str
         Access-URL, see below for supported formats.
+    store_cls : Optional[Type[KeyValueStore]]
+        The class of the store to create.
+        If the URL scheme doesn't match the class, a ValueError is raised.
 
     Returns
     -------
@@ -52,7 +59,118 @@ def get_store_from_url(url: str) -> KeyValueStore:
     json_b64_encoded = base64.urlsafe_b64encode(b).decode()
 
     """
-    return get_store(**url2dict(url))
+    from minimalkv._hstores import (
+        HAzureBlockBlobStore,
+        HDictStore,
+        HFilesystemStore,
+        HGoogleCloudStore,
+        HS3FSStore,
+    )
+    from minimalkv.fs import FilesystemStore
+    from minimalkv.memory import DictStore
+    from minimalkv.memory.redisstore import RedisStore
+    from minimalkv.net.azurestore import AzureBlockBlobStore
+    from minimalkv.net.gcstore import GoogleCloudStore
+    from minimalkv.net.s3fsstore import S3FSStore
+
+    scheme_to_store: Dict[str, Type[KeyValueStore]] = {
+        "azure": AzureBlockBlobStore,
+        "hazure": HAzureBlockBlobStore,
+        "s3": S3FSStore,
+        "hs3": HS3FSStore,
+        "boto": HS3FSStore,
+        "gcs": GoogleCloudStore,
+        "hgcs": HGoogleCloudStore,
+        "fs": FilesystemStore,
+        "file": FilesystemStore,
+        "hfs": HFilesystemStore,
+        "hfile": HFilesystemStore,
+        "filesystem": HFilesystemStore,
+        "memory": DictStore,
+        "hmemory": HDictStore,
+        "redis": RedisStore,
+    }
+
+    parsed_url = urisplit(url)
+    # Wrappers can be used to add functionality to a store, e.g. encryption.
+    # Wrappers are separated by `+` and can be specified in two ways:
+    # 1. As part of the scheme, e.g. "s3+readonly://..." (old style)
+    # 2. As the fragment, e.g. "s3://...#wrap:readonly" (new style)
+    wrappers = extract_wrappers(parsed_url)
+
+    # Remove wrappers from scheme
+    scheme_parts = parsed_url.getscheme().split("+")
+    # pop off the type of the store
+    scheme = scheme_parts[0]
+
+    if scheme not in scheme_to_store:
+        raise ValueError(f'Unknown storage type "{scheme}"')
+
+    store_cls_from_url = scheme_to_store[scheme]
+    if store_cls is not None and store_cls_from_url != store_cls:
+        raise ValueError(
+            f"URL scheme {scheme} does not match store class {store_cls.__name__}"
+        )
+
+    query_listdict: Dict[str, List[str]] = parsed_url.getquerydict()
+    # We will just use the last occurrence for each key
+    query = {k: v[-1] for k, v in query_listdict.items()}
+
+    store = store_cls_from_url.from_parsed_url(parsed_url, query)
+
+    # apply wrappers/decorators:
+    from minimalkv._store_decoration import decorate_store
+
+    wrapped_store = reduce(decorate_store, wrappers, store)
+
+    return wrapped_store
+
+
+def extract_wrappers(parsed_url: SplitResult) -> List[str]:
+    """
+    Extract wrappers from a parsed URL.
+
+    Wrappers allow you to add additional functionality to a store, e.g. encryption.
+    Wrappers are specified in the fragment part of the URL, e.g. "s3://...#wrap:readonly+urlencode"
+
+    Wrappers can also be specified as part of the scheme, e.g. "s3+readonly+urlencode://...".
+    This is deprecated and will be removed in a future version.
+
+    Parameters
+    ----------
+    parsed_url: SplitResult
+        The parsed URL.
+
+    Returns
+    -------
+    wrappers: List[str]
+        The list of wrappers.
+    """
+    # split off old-style wrappers, if any:
+    parts = parsed_url.getscheme().split("+")
+    # pop off the type of the store
+    parts.pop(0)
+    old_wrappers = list(reversed(parts))
+
+    # find new-style wrappers, if any:
+    fragment = parsed_url.getfragment()
+    fragments = fragment.split("#") if fragment else []
+    wrap_spec = list(filter(lambda s: s.startswith("wrap:"), fragments))
+    if wrap_spec:
+        fragment_wrappers = wrap_spec[-1].partition("wrap:")[
+            2
+        ]  # remove the 'wrap:' part
+        new_wrappers = list(fragment_wrappers.split("+"))
+    else:
+        new_wrappers = []
+
+    # can't have both:
+    if old_wrappers and new_wrappers:
+        raise ValueError(
+            "Adding store wrappers via store type as well as via wrap parameter are not allowed. Preferably use wrap."
+        )
+
+    return old_wrappers + new_wrappers
 
 
 def get_store(
@@ -118,12 +236,20 @@ def get_store(
         Key value store of type ``type`` as described in ``kwargs`` parameters.
 
     """
-    from minimalkv._store_creation import create_store
+    warn(
+        """
+        get_store will be removed in the next major release.
+        If you want to create a KeyValueStore from a URL, use get_store_from_url.
+        """,
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    from minimalkv._old_store_creation import create_store
     from minimalkv._store_decoration import decorate_store
 
     # split off old-style wrappers, if any:
     parts = type.split("+")
-    type = parts.pop(-1)
+    type = parts.pop(0)
     decorators = list(reversed(parts))
 
     # find new-style wrappers, if any:

minimalkv/_hstores.py

@@ -5,8 +5,10 @@
 from minimalkv.memory import DictStore
 from minimalkv.memory.redisstore import RedisStore
 from minimalkv.net.azurestore import AzureBlockBlobStore
+from minimalkv.net.boto3store import Boto3Store
 from minimalkv.net.botostore import BotoStore
 from minimalkv.net.gcstore import GoogleCloudStore
+from minimalkv.net.s3fsstore import S3FSStore
 
 
 class HDictStore(ExtendedKeyspaceMixin, DictStore):  # noqa D
@@ -39,6 +41,14 @@ def size(self, key: str) -> bytes:
         return k.size
 
 
+class HS3FSStore(ExtendedKeyspaceMixin, S3FSStore):  # noqa D
+    pass
+
+
+class HBoto3Store(ExtendedKeyspaceMixin, Boto3Store):  # noqa D
+    pass
+
+
 class HGoogleCloudStore(ExtendedKeyspaceMixin, GoogleCloudStore):  # noqa D
     pass
 

minimalkv/_key_value_store.py

@@ -1,6 +1,8 @@
 from io import BytesIO
 from types import TracebackType
-from typing import IO, Iterable, Iterator, List, Optional, Type, Union
+from typing import IO, Dict, Iterable, Iterator, List, Optional, Type, Union
+
+from uritools import SplitResult
 
 from minimalkv._constants import VALID_KEY_RE
 from minimalkv._mixins import UrlMixin
@@ -98,7 +100,7 @@ def get_file(self, key: str, file: Union[str, IO]) -> str:
         implement a specialized function if data needs to be written to disk or streamed.
 
         If ``file`` is a string, contents of ``key`` are written to a newly created file
-        with the filename ``file``. Otherwise the data will be written using the
+        with the filename ``file``. Otherwise, the data will be written using the
         ``write`` method of ``file``.
 
         Parameters
@@ -462,6 +464,48 @@ def __exit__(
         """
         self.close()
 
+    @classmethod
+    def from_url(cls, url: str) -> "KeyValueStore":
+        """Create a Store from a URL.
+
+        Parameters
+        ----------
+        url : str
+            URL to create store from.
+
+        Returns
+        -------
+        store : KeyValueStore
+            Store created from URL.
+        """
+        from minimalkv import get_store_from_url
+
+        store = get_store_from_url(url, store_cls=cls)
+        if not isinstance(store, cls):
+            raise ValueError(f"Expected {cls}, got {type(store)}")
+        return store
+
+    @classmethod
+    def from_parsed_url(
+        cls, parsed_url: SplitResult, query: Dict[str, str]
+    ) -> "KeyValueStore":
+        """
+        Build a KeyValueStore from a parsed URL.
+
+        Parameters
+        ----------
+        parsed_url: SplitResult
+            The parsed URL.
+        query: Dict[str, str]
+            Query parameters from the URL.
+
+        Returns
+        -------
+        store : KeyValueStore
+            The created KeyValueStore.
+        """
+        raise NotImplementedError
+
 
 class UrlKeyValueStore(UrlMixin, KeyValueStore):
     """Class is deprecated. Use the :class:`.UrlMixin` instead.