Move storefact code into place

LICENSE

@@ -1,3 +1,30 @@
+Copyright (c) 2021, The minimal kv contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+DEALINGS IN THE SOFTWARE.
+
+--
+The original simplekv source had the following license:
+
 Copyright (c) 2015 Marc Brinkmann
 
 Permission is hereby granted, free of charge, to any person obtaining a
@@ -16,4 +43,30 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+
+-- 
+The original storefact source had the following license:
+
+Copyright (c) 2015-2017, Blue Yonder GmbH
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 DEALINGS IN THE SOFTWARE.

docs/index.rst

@@ -46,6 +46,50 @@ Note that by changing the first two lines to::
 you could use the code exactly the same way, this time storing data inside a
 Redis database.
 
+Store factories (from configurations)
+=====================================
+
+There are two possibilities to generate stores from configurations:
+
+1) Use a dictionary with configuration data (e.g. loaded from an ini file)
+
+.. code-block:: python
+
+    from minimalkv import get_store
+
+    params = {
+        'account_name': 'test',
+        'account_key': 'XXXsome_azure_account_keyXXX',
+        'container': 'my-azure-container',
+    }
+    store = get_store('azure', **params)
+    store.put(u'key', b'value')
+    assert store.get(u'key') == b'value'
+
+2) Use an URL to specify the configuration
+
+.. code-block:: python
+
+    from minimalkv import get_store_from_url, get_store
+
+    store = get_store_from_url('azure://test:XXXsome_azure_account_keyXXX@my-azure-container')
+    store.put(u'key', b'value')
+    assert store.get(u'key') == b'value'
+
+URL and store types:
+
+* In memory: :code:`memory://` and :code:`hmemory://`.
+* Redis: :code:`redis://[[password@]host[:port]][/db]` and :code:`hredis://[[password@]host[:port]][/db]`
+* Filesystem: :code:`fs://` and :code:`hfs://`
+* Amazon S3: :code:`s3://access_key:secret_key@endpoint/bucket[?create_if_missing=true]` and :code:`hs3://access_key:secret_key@endpoint/bucket[?create_if_missing=true]`
+* Azure Blob Storage (:code:`azure://` and :code:`hazure://`):
+    * with storage account key: :code:`azure://account_name:account_key@container[?create_if_missing=true][?max_connections=2]`
+    * with SAS token: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)]`
+    * with SAS and additional parameters: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024]`
+
+Storage URLs starting with a :code:`h` indicate extended allowed characters. This allows the usage of slashes and spaces in blob names.
+URL options with :code:`[]` are optional and the :code:`[]` need to be removed.
+
 
 Why you should  use minimalkv
 ============================

environment.yml

@@ -21,3 +21,4 @@ dependencies:
   - pymysql
   - pre-commit
   - setuptools-scm
+  - uritools

minimalkv/__init__.py

@@ -3,10 +3,10 @@
 
 import re
 from io import BytesIO
+from functools import reduce
 
 from ._compat import key_type
-
-__version__ = "0.14.1"
+from ._urls import url2dict
 
 VALID_NON_NUM = r"""\`\!"#$%&'()+,-.<=>?@[]^_{}~"""
 VALID_KEY_REGEXP = "^[%s0-9a-zA-Z]+$" % re.escape(VALID_NON_NUM)
@@ -503,3 +503,102 @@ def _move(self, source, dest):
         self._copy(source, dest)
         self._delete(source)
         return dest
+
+
+def get_store_from_url(url):
+    """
+    Take a URL and return a simplekv store according to the parameters in the URL.
+
+    .. note::
+       User credentials like secret keys have to be percent-encoded before they can be
+       used in a URL (see *azure* and *s3* store types), since they can contain characters
+       that are not valid in this part of a URL, like forward-slashes.
+
+
+       You can use Python to percent-encode your secret key on the commandline like so::
+
+           $ python -c "import urllib; print urllib.quote_plus('''dead/beef''')"
+           dead%2Fbeef
+
+    :param url: Access-URL, see below for supported forms
+    :return: Parameter dictionary suitable for get_store()
+
+    Store types and URL forms:
+
+    * DictStore: ``memory://``
+    * RedisStore: ``redis://[[password@]host[:port]][/db]``
+    * FilesystemStore: ``fs://path``
+    * BotoStore ``s3://access_key:secret_key@endpoint/bucket[?create_if_missing=true]``
+    * AzureBlockBlockStorage: ``azure://account_name:account_key@container[?create_if_missing=true]``
+    * AzureBlockBlockStorage (SAS): ``azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false``
+    * AzureBlockBlockStorage (SAS): ``azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)]``
+    * AzureBlockBlockStorage (SAS): ``azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024]``
+    """
+    return get_store(**url2dict(url))
+
+
+def get_store(type, create_if_missing=True, **params):
+    """Return a storage object according to the `type` and additional parameters.
+
+    The *type* must be one of the types below, where each allows
+    different parameters:
+
+    * ``"azure"``: Returns a ``simplekv.azure.AzureBlockBlobStorage``. Parameters are
+      ``"account_name"``, ``"account_key"``, ``"container"``, ``"use_sas"`` and ``"create_if_missing"`` (default: ``True``).
+      ``"create_if_missing"`` has to be ``False`` if ``"use_sas"`` is set. When ``"use_sas"`` is set,
+      ``"account_key"`` is interpreted as Shared Access Signature (SAS) token.FIRE
+      ``"max_connections"``: Maximum number of network connections used by one store (default: ``2``).
+      ``"socket_timeout"``: maximum timeout value in seconds (socket_timeout: ``200``).
+      ``"max_single_put_size"``: max_single_put_size is the largest size upload supported in a single put call.
+      ``"max_block_size"``: maximum block size is maximum size of the blocks(maximum size is <= 100MB)
+    * ``"s3"``: Returns a plain ``simplekv.net.botostore.BotoStore``.
+      Parameters must include ``"host"``, ``"bucket"``, ``"access_key"``, ``"secret_key"``.
+      Optional parameters are
+
+       - ``"force_bucket_suffix"`` (default: ``True``). If set, it is ensured that
+         the bucket name ends with ``-<access_key>``
+         by appending this string if necessary;
+         If ``False``, the bucket name is used as-is.
+       - ``"create_if_missing"`` (default: ``True`` ). If set, creates the bucket if it does not exist;
+         otherwise, try to retrieve the bucket and fail with an ``IOError``.
+    * ``"hs3"`` returns a variant of ``simplekv.net.botostore.BotoStore`` that allows "/" in the key name.
+      The parameters are the same as for ``"s3"``
+    * ``"fs"``: Returns a ``simplekv.fs.FilesystemStore``. Specify the base path as "path" parameter.
+    * ``"hfs"`` returns a variant of ``simplekv.fs.FilesystemStore``  that allows "/" in the key name.
+      The parameters are the same as for ``"file"``.
+    * ``"memory"``: Returns a DictStore. Doesn't take any parameters
+    * ``"redis"``: Returns a RedisStore. Constructs a StrictRedis using params as kwargs.
+      See StrictRedis documentation for details.
+
+    :param str type: Type of storage to open, with optional storage decorators
+    :param boolean create_if_missing: Create the "root" of the storage (Azure container, parent directory, S3 bucket, etc.).
+      Has no effect for stores where this makes no sense, like `redis` or `memory`.
+    :param kwargs: Parameters specific to the Store-class"""
+    from ._store_creation import create_store
+    from ._store_decoration import decorate_store
+
+    # split off old-style wrappers, if any:
+    parts = type.split('+')
+    type = parts.pop(-1)
+    decorators = list(reversed(parts))
+
+    # find new-style wrappers, if any:
+    wrapspec = params.pop('wrap', '')
+    wrappers = list(wrapspec.split('+')) if wrapspec else []
+
+    # can't have both:
+    if wrappers:
+        if decorators:
+            raise ValueError('Adding store wrappers via store type as well as via wrap parameter are not allowed. Preferably use wrap.')
+        decorators = wrappers
+
+    # create_if_missing is a universal parameter, so it's part of the function signature
+    # it can be safely ignored by stores where 'creating' makes no sense.
+    params['create_if_missing'] = create_if_missing
+
+    store = create_store(type, params)
+
+    # apply wrappers/decorators:
+    wrapped_store = reduce(decorate_store, decorators, store)
+
+    return wrapped_store

storefact/_boto.py → minimalkv/_boto.py

@@ -1,9 +1,3 @@
-# -*- coding: utf-8 -*-
-""""""
-
-from __future__ import (absolute_import, division, print_function)
-
-
 def _get_s3bucket(host, bucket, access_key, secret_key, force_bucket_suffix=True, create_if_missing=True):
     from boto.s3.connection import S3Connection, OrdinaryCallingFormat, S3ResponseError
 

storefact/_hstores.py → minimalkv/_hstores.py

@@ -1,15 +1,11 @@
-# -*- coding: utf-8 -*-
-""""""
-
-from __future__ import (absolute_import, division, print_function)
 import os
 
-from simplekv.contrib import ExtendedKeyspaceMixin
-from simplekv.fs import FilesystemStore
-from simplekv.memory import DictStore
-from simplekv.memory.redisstore import RedisStore
-from simplekv.net.azurestore import AzureBlockBlobStore
-from simplekv.net.botostore import BotoStore
+from minimalkv.contrib import ExtendedKeyspaceMixin
+from minimalkv.fs import FilesystemStore
+from minimalkv.memory import DictStore
+from minimalkv.memory.redisstore import RedisStore
+from minimalkv.net.azurestore import AzureBlockBlobStore
+from minimalkv.net.botostore import BotoStore
 
 
 class HDictStore(ExtendedKeyspaceMixin, DictStore):

storefact/_store_creation.py → minimalkv/_store_creation.py

@@ -1,12 +1,7 @@
-# -*- coding: utf-8 -*-
-""""""
-
-from __future__ import (absolute_import, division, print_function)
-
 import os
 import os.path
 
-from simplekv.fs import FilesystemStore
+from minimalkv.fs import FilesystemStore
 
 
 def create_store(type, params):
@@ -30,7 +25,7 @@ def create_store(type, params):
 
 
 def _create_store_azure(type, params):
-    from simplekv.net.azurestore import AzureBlockBlobStore
+    from minimalkv.net.azurestore import AzureBlockBlobStore
     from ._hstores import HAzureBlockBlobStore
 
     conn_string = params.get('connection_string', _build_azure_url(**params))
@@ -71,7 +66,7 @@ def _create_store_hs3(type, params):
 
 
 def _create_store_s3(type, params):
-    from simplekv.net.botostore import BotoStore
+    from minimalkv.net.botostore import BotoStore
     from ._boto import _get_s3bucket
     return BotoStore(_get_s3bucket(**params))
 
@@ -90,7 +85,7 @@ def _create_store_fs(type, params):
 
 
 def _create_store_mem(type, params):
-    from simplekv.memory import DictStore
+    from minimalkv.memory import DictStore
     return DictStore()
 
 
@@ -100,7 +95,7 @@ def _create_store_hmem(type, params):
 
 
 def _create_store_redis(type, params):
-    from simplekv.memory.redisstore import RedisStore
+    from minimalkv.memory.redisstore import RedisStore
     from redis import StrictRedis
     r = StrictRedis(**params)
     return RedisStore(r)

storefact/_store_decoration.py → minimalkv/_store_decoration.py

@@ -1,9 +1,4 @@
-# -*- coding: utf-8 -*-
-""""""
-
-from __future__ import (absolute_import, division, print_function)
-
-from simplekv.decorator import (URLEncodeKeysDecorator, ReadOnlyDecorator)
+from minimalkv.decorator import (URLEncodeKeysDecorator, ReadOnlyDecorator)
 
 
 def decorate_store(store, decoratorname):

minimalkv/fs.py

@@ -5,8 +5,8 @@
 import os.path
 import shutil
 
-from . import CopyMixin, KeyValueStore, UrlMixin
-from ._compat import text_type, url_quote
+from minimalkv import CopyMixin, KeyValueStore, UrlMixin
+from minimalkv._compat import text_type, url_quote
 
 
 class FilesystemStore(KeyValueStore, UrlMixin, CopyMixin):

setup.py

@@ -19,12 +19,12 @@
     author="Data Engineering Collective",
     author_email="minimalkv@uwekorn.com",
     url="https://github.com/data-engineering-collective/minimalkv",
-    license="MIT",
+    license="BSD-3-clause",
     packages=find_packages(exclude=["test"]),
-    install_requires=[],
+    install_requires=["uritools"],
     python_requires=">=3.7",
     classifiers=[
-        "License :: OSI Approved :: MIT License",
+        "License :: OSI Approved :: BSD License",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",