Merge branch 'master' into pyup-initial-update

* master:
  update to working version of pairtree (0.8.1)
  update versions
  update pairtree to official release
  updated history.rst and setup.py
  fixing coverage
  Update requirements.txt
  update requirements because coveralls asks me to
  renamed function
  updating config
  fixing tests
  trying to add some checks to compensate for silent fail in pairtree. (we need a PR for this)
  fixing python2.7 only issue (ints will be autopromoted when going for a long)
  while waiting for the published pypi package, we fetch directly from github
  Update setup.py
  Update setup.py
  Update HISTORY.rst
  Update requirements.txt
  create pyup.io config file

Conflicts:
	requirements-dev.txt
	requirements.txt

.pyup.yml

@@ -0,0 +1,4 @@
+# autogenerated pyup.io config file 
+# see https://pyup.io/docs/configuration/ for all available options
+
+schedule: every month

.travis.yml

@@ -2,6 +2,7 @@ language: python
 sudo: false
 python:
   - "2.7"
+  - "3.5"
 env:
   - PROJECT=augeias
 notifications:

HISTORY.rst

@@ -1,3 +1,18 @@
+0.4.1 (26-10-2017)
+------------------
+
+- pairtree update to 0.8.1
+
+0.4.0 (24-10-2017)
+------------------
+
+- Added python 3 compatibility
+
+0.3.1 (12-10-2017)
+------------------
+
+- Fix pairtree dependency==0.7.1-T
+
 0.3.0 (22-08-2017)
 ------------------
 

augeias/stores/PairTreeFileSystemStore.py

@@ -9,6 +9,7 @@
 import os
 import magic
 import datetime
+import sys
 
 
 class PairTreeFileSystemStore(IStore):
@@ -58,7 +59,7 @@ def get_object_info(self, container_key, object_key):
         return {
             'time_last_modification': datetime.datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
             'size': file_stat.st_size,
-            'mime': magic.from_buffer(open(file_path).read(1048576), mime=True) or 'application/octet-stream'
+            'mime': magic.from_buffer(open(file_path, 'rb').read(1048576), mime=True) or 'application/octet-stream'
         }
 
     def create_object(self, container_key, object_key, object_data):
@@ -70,6 +71,7 @@ def create_object(self, container_key, object_key, object_data):
         :param str object_data: The data for the object to create.
         :raises augeias.stores.error.NotFoundException: When the container could not be found.
         '''
+        _validate_data(object_data)
         container = self._get_container(container_key)
         container.add_bytestream(object_key, object_data)
 
@@ -82,6 +84,7 @@ def update_object(self, container_key, object_key, object_data):
         :param str object_data: New data for the object.
         :raises augeias.stores.error.NotFoundException: When the object or container could not be found.
         '''
+        _validate_data(object_data)
         container = self.store.get_object(container_key, False)
         container.add_bytestream(object_key, object_data)
 
@@ -130,3 +133,20 @@ def delete_container(self, container_key):
             self.store.delete_object(container_key)
         except ObjectNotFoundException:
             raise NotFoundException
+
+
+def _is_allowed_data(data):  # pragma: no cover
+    # not exhaustive.
+    python_version = sys.version_info.major
+    if python_version < 3 and isinstance(data, unicode):
+        return False
+
+    if python_version >= 3 and isinstance(data, str):
+        return False
+
+    return True
+
+
+def _validate_data(data):
+    if not _is_allowed_data(data):
+        raise IOError('Data type is not allowed')

augeias/views.py

@@ -183,9 +183,9 @@ def delete_container(self):
 # HELPERS
 
 
-def _is_long(s):
+def _is_integer(s):
     try:
-        long(s)
+        int(s)  # python 2.7 'auto-promotes' int to long if required
         return True
     except ValueError:
         return False
@@ -199,9 +199,9 @@ def _get_object_data(request):
 
 
 def _get_object_data_from_stream(request):
-    if 'Content-Length' not in request.headers or not _is_long(request.headers['Content-Length']):
+    if 'Content-Length' not in request.headers or not _is_integer(request.headers['Content-Length']):
         raise HTTPLengthRequired
-    content_length = long(request.headers['Content-Length'])
+    content_length = int(request.headers['Content-Length'])
     object_data = request.body_file
     if content_length == 0:
         raise HTTPBadRequest('body is empty')

requirements-dev.txt

@@ -19,7 +19,7 @@ sphinxcontrib-httpdomain==1.5.0
 sphinx_rtd_theme==0.2.4
 
 # waitress
-waitress==1.0.2
+waitress==1.1.0
 
 # Linting
 flake8==3.4.1

requirements.txt

@@ -3,7 +3,8 @@ pyramid==1.9.1
 pyramid_rewrite==0.2
 
 # pairtree
-pairtree>=0.5.8
+ez_setup==0.9
+pairtree==0.8.1
 python-magic==0.4.13
 
 venusian==1.1.0

setup.py

@@ -10,13 +10,14 @@
 
 requires = [
     'pyramid',
-    'pairtree',
+    'ez_setup==0.9',
+    'pairtree==0.8.1',
     'python-magic',
     'pyramid_rewrite'
     ]
 
 setup(name='augeias',
-      version='0.3.0',
+      version='0.4.1',
       description='Augeias. Stores your files.',
       long_description=README + '\n\n' + HISTORY,
       classifiers=[

tests/test_functional.py

@@ -9,10 +9,13 @@
 from augeias.collections import Collection
 from augeias.stores.PairTreeFileSystemStore import PairTreeFileSystemStore
 import json
+import sys
 
 here = os.path.dirname(__file__)
 settings = get_appsettings(os.path.join(here, 'conf_test.ini'))
 
+python_version = sys.version_info.major
+
 
 def collections_include(config, store_dir):
     test_collection = Collection(name='TEST_COLLECTION', object_store=PairTreeFileSystemStore(store_dir))
@@ -50,8 +53,8 @@ def test_create_container(self):
         res = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', res.status)
         self.assertIn('application/json', res.headers['Content-Type'])
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID', res.body)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID', res.text)
 
 
     def test_add_object(self):
@@ -65,9 +68,9 @@ def test_add_object(self):
         res = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', res.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      res.body)
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn('200x300', res.body)
+                      res.text)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn('200x300', res.text)
 
     def test_add_object_content_length_errors(self):
         # create a container
@@ -90,14 +93,14 @@ def test_get_object(self):
         cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', cres.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID',
-                      cres.body)
+                      cres.text)
         testdata = os.path.join(here, '../', 'fixtures/kasteel.jpg')
         with open(testdata, 'rb') as f:
             bdata = f.read()
         file_size = len(bdata)
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         self.assertEqual('200 OK', ores.status)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300')
@@ -110,13 +113,13 @@ def test_get_object_info(self):
         cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', cres.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID',
-                      cres.body)
+                      cres.text)
         testdata = os.path.join(here, '../', 'fixtures/kasteel.jpg')
         with open(testdata, 'rb') as f:
             bdata = f.read()
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         self.assertEqual('200 OK', ores.status)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300/meta')
@@ -154,7 +157,7 @@ def test_list_object_keys_for_container(self):
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', res.status)
-        l = ast.literal_eval(res.body)
+        l = ast.literal_eval(res.text)
         l = [i.strip() for i in l]
         self.assertTrue('200x300' in l and '400x600' in l)
 
@@ -168,7 +171,7 @@ def test_update_object(self):
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', ores.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
 
         testdata = os.path.join(here, '../', 'fixtures/brug.jpg')
         with open(testdata, 'rb') as f:
@@ -203,7 +206,7 @@ def test_copy_object(self):
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', ores.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         cres2 = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID2')
         self.assertEqual('200 OK', cres2.status)
         json_data = json.dumps({
@@ -278,8 +281,8 @@ def test_delete_object(self):
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
                       res.json_body['uri'])
         self.assertEqual('200 OK', res.status)
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn('200x300', res.body)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn('200x300', res.text)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', status=404,
                                expect_errors=True)
@@ -305,7 +308,7 @@ def test_create_container_with_id(self):
         self.assertEqual('201 Created', res.status)
         uuid4hex = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z', re.I)
         container_key = res.json_body['container_key']
-        if isinstance(container_key, unicode):
+        if python_version < 3 and isinstance(container_key, unicode):
             container_key = str(container_key)
         self.assertTrue(uuid4hex.match(container_key))
         print(res.json_body['uri'])
@@ -320,7 +323,7 @@ def test_create_object_with_id(self):
         self.assertEqual('201 Created', res.status)
         uuid4hex = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z', re.I)
         object_key = res.json_body['object_key']
-        if isinstance(object_key, unicode):
+        if python_version < 3 and isinstance(object_key, unicode):
             object_key = str(object_key)
         self.assertTrue(uuid4hex.match(object_key))
         print(res.json_body['uri'])

tests/test_stores.py

@@ -2,7 +2,7 @@
 import unittest
 import tempdir
 from augeias.stores.CephStore import CephStore
-from augeias.stores.PairTreeFileSystemStore import PairTreeFileSystemStore
+from augeias.stores.PairTreeFileSystemStore import PairTreeFileSystemStore, _is_allowed_data, _validate_data
 from augeias.stores.error import NotFoundException
 
 
@@ -22,12 +22,12 @@ def test_usage_scenario(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         object_list = self.store.list_object_keys_for_container(container_key)
         self.assertEqual(1, len(object_list))
         self.assertEqual(object_key, object_list[0])
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('some test data', object_value)
+        self.assertEqual(b'some test data', object_value)
         self.store.delete_object(container_key, object_key)
         object_list = self.store.list_object_keys_for_container(container_key)
         self.assertEqual(0, len(object_list))
@@ -70,25 +70,25 @@ def test_update_scenario(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('some test data', object_value)
-        self.store.update_object(container_key, object_key, 'updated data')
+        self.assertEqual(b'some test data', object_value)
+        self.store.update_object(container_key, object_key, b'updated data')
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('updated data', object_value)
+        self.assertEqual(b'updated data', object_value)
 
     def test_delete_nonexisting(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         self.assertRaises(NotFoundException, self.store.delete_object, container_key, 'nogo')
 
     def test_add_object_to_nonexisting_container(self):
         error_raised = False
         self.store.create_container('x')
         try:
-            self.store.create_object('xx', '253', 'some test data')
+            self.store.create_object('xx', '253', b'some test data')
         except NotFoundException:
             error_raised = True
         self.assertTrue(error_raised)
@@ -105,6 +105,13 @@ def test_delete_nonexisting_container(self):
             error_raised = True
         self.assertTrue(error_raised)
 
+    def test_is_allowed_data(self):
+        self.assertFalse(_is_allowed_data(u'foo'))
+        self.assertTrue(_is_allowed_data(b'data'))
+
+    def test_validate_data(self):
+        self.assertRaises(IOError, _validate_data, u'foo')
+
 
 class TestCephStore(unittest.TestCase):
     '''series of tests to check the implementation of the CephStore.