Use azurite for azure blob storage tests

.gitignore

@@ -97,5 +97,3 @@ ENV/
 
 # PyCharm
 .idea/
-
-azure_credentials.ini

.travis.yml

@@ -23,6 +23,7 @@ matrix:
         - TOXENV=py36-old-azure-storage-blob
 
 services:
+- docker
 - redis-server
 - mongodb
 - postgresql
@@ -32,11 +33,11 @@ install: pip install tox coveralls
 
 before_script:
 - bash .travis/start_minio.sh
+- docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 &
 - psql -c 'create database simplekv_test;' -U postgres
 - psql -c 'ALTER ROLE travis CONNECTION LIMIT -1;' -U postgres
 - mysql -e 'create database simplekv_test;'
 - mysql -e 'set global max_connections = 200;'
-- '[ -z "$encrypted_dea9dfb12f4a_key" ] || openssl aes-256-cbc -K $encrypted_dea9dfb12f4a_key -iv $encrypted_dea9dfb12f4a_iv -in azure_credentials.ini.enc -out azure_credentials.ini -d'
 
 script: tox
 

azure_credentials.ini

@@ -0,0 +1,12 @@
+# Specify the azure account to use in the tests.
+# The default assumes you are running azurite (https://github.com/Azure/Azurite).
+# To skip the azure store tests, remove the ``account_name``.
+#
+# Note that only the first config section is parsed.
+
+[azure-tests]
+account_name=devstoreaccount1
+account_key=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+# these are not needed for a real azure blob store accounts; only set them for azurite:
+protocol=http
+endpoint=http://127.0.0.1:10000/devstoreaccount1

docs/azure.rst

@@ -27,17 +27,26 @@ Here is a short example:
    print store.get(u'some-key')
 
 
-Unit testing
-============
+Testing
+=======
 
-The unit-tests for the azure backend are either run by travis on the main repo, or if you provide
-your own credentials via a `azure_credentials.ini`` file in the root folder of the project.
-An example looks like this:
+The tests for the azure backend either
 
-::
+ * use a real azure blob store account or
+ * use the `Azurite <https://github.com/Azure/Azurite>`_ blob storage emulator
+
+The travis tests use the second method.
+
+To test with a real blob store account, edit the file ``azure_credentials.ini``
+s.t. the first config section contains the actual account_name and account_key
+of your test account.
+
+To test against a locally running azurite instance make sure to start azurite::
+
+ docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 &
+
+before running the tests.
 
-  [my-azure-test-account]
-  account_name=my_account_name
-  account_key=AZURE_TEST_KEY
+To skip the tests of the azure backend, comment out the ``account_name`` in the ``azure_credentials.ini`` file.
 
 .. autoclass:: simplekv.net.azurestore.AzureBlockBlobStore

tests/test_azure_store.py

@@ -7,34 +7,30 @@
 import pytest
 from base64 import b64encode
 
-pytest.importorskip('azure.storage.blob')
+asb = pytest.importorskip('azure.storage.blob')
 
 
-def load_azure_credentials():
-    # loaded from the same place as tox.ini. here's a sample
-    #
-    # [my-azure-storage-account]
-    # account_name=foo
-    # account_key=bar
+def get_azure_conn_string():
     cfg_fn = 'azure_credentials.ini'
-
     parser = ConfigParser()
     result = parser.read(cfg_fn)
     if not result:
         pytest.skip('file {} not found'.format(cfg_fn))
 
     for section in parser.sections():
-        return {
-            'account_name': parser.get(section, 'account_name'),
-            'account_key': parser.get(section, 'account_key'),
-        }
-
-
-def create_azure_conn_string(credentials):
-    account_name = credentials['account_name']
-    account_key = credentials['account_key']
-    fmt_str = 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'
-    return fmt_str.format(account_name, account_key)
+        account_name = parser.get(section, 'account_name', fallback=None)
+        if account_name is None:
+            pytest.skip("no 'account_name' found in file {}".format(cfg_fn))
+
+        account_key = parser.get(section, 'account_key')
+        protocol = parser.get(section, 'protocol', fallback='https')
+        endpoint = parser.get(section, 'endpoint', fallback=None)
+        conn_string = 'DefaultEndpointsProtocol={};AccountName={};AccountKey={}'.format(
+            protocol, account_name, account_key
+        )
+        if endpoint is not None:
+            conn_string += ';BlobEndpoint={}'.format(endpoint)
+        return conn_string
 
 
 def _delete_container(conn_string, container):
@@ -61,7 +57,7 @@ class TestAzureStorage(BasicStore, OpenSeekTellStore):
     @pytest.fixture
     def store(self):
         container = str(uuid())
-        conn_string = create_azure_conn_string(load_azure_credentials())
+        conn_string = get_azure_conn_string()
         yield AzureBlockBlobStore(conn_string=conn_string, container=container,
                                   public=False)
         _delete_container(conn_string, container)
@@ -70,19 +66,23 @@ def store(self):
 class TestExtendedKeysAzureStorage(TestAzureStorage, ExtendedKeyspaceTests):
     @pytest.fixture
     def store(self):
+        azure_storage_blob_major_version = int(asb.__version__.split('.', 1)[0])
+        conn_string = get_azure_conn_string()
+        use_azurite = 'http://127.0.0.1:10000/devstoreaccount1' in conn_string
+        if use_azurite and azure_storage_blob_major_version < 12:
+            pytest.skip("Compatibility issues with azurite and azure-storage-blob<12")
+        container = str(uuid())
+
         class ExtendedKeysStore(ExtendedKeyspaceMixin, AzureBlockBlobStore):
             pass
-
-        container = str(uuid())
-        conn_string = create_azure_conn_string(load_azure_credentials())
         yield ExtendedKeysStore(conn_string=conn_string,
                                 container=container, public=False)
         _delete_container(conn_string, container)
 
 
 def test_azure_setgetstate():
     container = str(uuid())
-    conn_string = create_azure_conn_string(load_azure_credentials())
+    conn_string = get_azure_conn_string()
     store = AzureBlockBlobStore(conn_string=conn_string, container=container)
     store.put(u'key1', b'value1')
 
@@ -116,7 +116,7 @@ def test_azure_special_args():
     # For azure-storage-blob 12,
     # test that the special arguments `max_block_size` and
     # `max_single_put_size` propagate to the constructed ContainerClient
-    conn_string = create_azure_conn_string(load_azure_credentials())
+    conn_string = get_azure_conn_string()
     MBS = 983645
     MSP = 756235
     abbs = AzureBlockBlobStore(
@@ -135,7 +135,7 @@ def test_azure_special_args():
 class TestAzureExceptionHandling(object):
     def test_missing_container(self):
         container = str(uuid())
-        conn_string = create_azure_conn_string(load_azure_credentials())
+        conn_string = get_azure_conn_string()
         store = AzureBlockBlobStore(conn_string=conn_string,
                                     container=container,
                                     create_if_missing=False)
@@ -145,9 +145,10 @@ def test_missing_container(self):
 
     def test_wrong_endpoint(self):
         container = str(uuid())
-        conn_string = create_azure_conn_string(load_azure_credentials())
-        conn_string += \
-            ";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;"
+        conn_string = get_azure_conn_string()
+        conn_settings = dict([s.split("=", 1) for s in conn_string.split(";") if s])
+        conn_settings['BlobEndpoint'] = 'https://host-does-not-exist/'
+        conn_string = ';'.join('{}={}'.format(key, value) for key, value in conn_settings.items())
         store = AzureBlockBlobStore(conn_string=conn_string,
                                     container=container,
                                     create_if_missing=False)
@@ -193,7 +194,7 @@ class TestChecksum(object):
     @pytest.fixture
     def store(self):
         container = str(uuid())
-        conn_string = create_azure_conn_string(load_azure_credentials())
+        conn_string = get_azure_conn_string()
 
         yield AzureBlockBlobStore(
             conn_string=conn_string,