Merge pull request #145 from roman-captural/abstract-filesystems

Rewrite of storage backend using fs

Dockerfile

@@ -3,7 +3,8 @@ FROM docker.io/debian:buster-slim
 # install imagetagger system dependencies
 RUN apt-get update && \
 	apt-get install --no-install-recommends -y g++ wget uwsgi-plugin-python3 python3 python3-pip node-uglify make git \
-	    python3-psycopg2 python3-ldap3 gettext gcc python3-dev python3-setuptools libldap2-dev libsasl2-dev nginx
+	    python3-psycopg2 python3-ldap3 python3-pkg-resources gettext gcc python3-dev python3-setuptools libldap2-dev \
+	    libsasl2-dev nginx
 
 # add requirements file
 WORKDIR /app/src

imagetagger/imagetagger/base/filesystem.py

@@ -0,0 +1,21 @@
+import fs
+import fs.base
+from django.conf import settings
+
+
+def root() -> fs.base.FS:
+    """Get the root filesystem object or create one by opening `settings.FS_URL`."""
+    if root._instance is None:
+        root._instance = fs.open_fs(settings.FS_URL)
+    return root._instance
+
+root._instance = None
+
+
+def tmp() -> fs.base.FS:
+    """Get the tmp filesystem object or create one by opening `settings.TMP_FS_URL`."""
+    if tmp._instance is None:
+        tmp._instance = fs.open_fs(settings.TMP_FS_URL)
+    return tmp._instance
+
+tmp._instance = None

imagetagger/imagetagger/images/management/commands/runzipdaemon.py

@@ -1,13 +1,14 @@
 import fasteners
-import os
+from fs import path
+from fs.zipfs import WriteZipFS
 from time import sleep
 
 from django.conf import settings
 from django.core.management import BaseCommand, CommandError
 from django.db.models import Q
-from zipfile import ZipFile
 
 from imagetagger.images.models import ImageSet
+from imagetagger.base.filesystem import root, tmp
 
 
 class Command(BaseCommand):
@@ -17,7 +18,7 @@ def handle(self, *args, **options):
         if not settings.ENABLE_ZIP_DOWNLOAD:
             raise CommandError('To enable zip download, set ENABLE_ZIP_DOWNLOAD to True in settings.py')
 
-        lock = fasteners.InterProcessLock(os.path.join(settings.IMAGE_PATH, 'zip.lock'))
+        lock = fasteners.InterProcessLock(path.combine(settings.IMAGE_PATH, 'zip.lock'))
         gotten = lock.acquire(blocking=False)
         if gotten:
             while True:
@@ -37,12 +38,16 @@ def _regenerate_zip(self, imageset):
             self.stderr.write('skipping regeneration of ready imageset {}'.format(imageset.name))
             return
 
-        with ZipFile(os.path.join(settings.IMAGE_PATH, imageset.tmp_zip_path()), 'w') as f:
-            for image in imageset.images.all():
-                f.write(image.path(), image.name)
-
-        os.rename(os.path.join(settings.IMAGE_PATH, imageset.tmp_zip_path()),
-                  os.path.join(settings.IMAGE_PATH, imageset.zip_path()))
+        tmp_zip_path = imageset.tmp_zip_path()
+        if not tmp().exists(path.dirname(tmp_zip_path)):
+            tmp().makedirs(path.dirname(tmp_zip_path))
+        with tmp().open(tmp_zip_path, 'wb') as zip_file:
+            with WriteZipFS(zip_file) as zip_fs:
+                for image in imageset.images.all():
+                    root().download(image.path(), zip_fs.openbin(image.name, 'w'))
+        with tmp().open(tmp_zip_path, 'rb') as zip_file:
+            root().upload(imageset.zip_path(), zip_file)
+        tmp().remove(tmp_zip_path)
 
         # Set state to ready if image set has not been set to invalid during regeneration
         ImageSet.objects.filter(pk=imageset.pk, zip_state=ImageSet.ZipState.PROCESSING) \

imagetagger/imagetagger/images/models.py

@@ -1,9 +1,9 @@
 from typing import Set
 
+from fs import path
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
-import os
 
 from imagetagger.users.models import Team
 
@@ -19,10 +19,10 @@ class Image(models.Model):
     height = models.IntegerField(default=600)
 
     def path(self):
-        return os.path.join(self.image_set.root_path(), self.filename)
+        return path.combine(self.image_set.root_path(), self.filename)
 
     def relative_path(self):
-        return os.path.join(self.image_set.path, self.filename)
+        return path.combine(self.image_set.path, self.filename)
 
     def delete(self, *args, **kwargs):
         self.image_set.zip_state = ImageSet.ZipState.INVALID
@@ -92,17 +92,23 @@ class ZipState:
     zip_state = models.IntegerField(choices=ZIP_STATES, default=ZipState.INVALID)
 
     def root_path(self):
-        return os.path.join(settings.IMAGE_PATH, self.path)
+        return path.combine(settings.IMAGE_PATH, self.path)
+
+    def tmp_path(self):
+        return path.combine(settings.TMP_IMAGE_PATH, self.path)
+
+    def relative_zip_path(self):
+        return path.combine(self.path, self.zip_name())
 
     def zip_path(self):
-        return os.path.join(self.path, self.zip_name())
+        return path.combine(self.root_path(), self.zip_name())
+
+    def tmp_zip_path(self):
+        return path.combine(self.tmp_path(), self.zip_name())
 
     def zip_name(self):
         return "imageset_{}.zip".format(self.id)
 
-    def tmp_zip_path(self):
-        return os.path.join(self.path, ".tmp." + self.zip_name())
-
     @property
     def image_count(self):
         if hasattr(self, 'image_count_agg'):