Merge pull request #455 from memeLab/pymarker-integration

[WIP] 249 Simplify Marker Upload

docker/base.Dockerfile

@@ -1,15 +1,12 @@
-FROM debian:buster-slim
+FROM python:3.9-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-        python3 \
-        python3-pip \
-        python3-setuptools \
         gettext \
         docutils-common
 COPY ./src/requirements.txt /src/requirements.txt
 
-RUN pip3 install --upgrade pip
-RUN pip3 install --no-cache-dir toolz
-RUN pip3 install --no-cache-dir -r /src/requirements.txt
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir toolz
+RUN pip install --no-cache-dir -r /src/requirements.txt
 
 RUN rm -rf ~/.cache/pip

docker/docker-compose.yml

@@ -13,10 +13,13 @@ services:
       - ../src/ARte/:/ARte/src/ARte/
       - ../locale/:/ARte/locale/
       - ./media/:/ARte/src/ARte/users/media/
+      - ../run.sh:/ARte/run.sh
     env_file:
       - ../src/.envs/.env
     depends_on:
       - postgres
+      - storage
+      - createbuckets
     command: ./run.sh
 
   postgres:
@@ -29,5 +32,34 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql/data
 
+  storage:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - storage:/storage
+    env_file:
+      - ../src/.envs/.env
+    command: server /storage --console-address ":9001"
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - storage
+    env_file:
+      - ../src/.envs/.env
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc config host add myminio $${MINIO_S3_ENDPOINT_URL} $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb myminio/$${AWS_STORAGE_BUCKET_NAME};
+      /usr/bin/mc policy set download myminio/$${AWS_STORAGE_BUCKET_NAME};
+      /usr/bin/mc mb myminio/$${AWS_PRIVATE_STORAGE_BUCKET_NAME};
+      /usr/bin/mc admin user add myminio $${MINIO_USER_ACCESS_KEY} $${MINIO_USER_SECRET_KEY};
+      /usr/bin/mc admin policy set myminio readwrite user=$${MINIO_USER_ACCESS_KEY};
+      exit 0;
+      "
+
 volumes:
   postgres_data:
+  storage:

src/.envs/.example

@@ -7,13 +7,21 @@ DEV_DB=False
 DEV_STATIC=True
 DEBUG_TOOLBAR=True
 DJANGO_SECRET_KEY=
+USE_MINIO=True
 
 ## Amazon AWS Variables
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-AWS_STORAGE_BUCKET_NAME=
-AWS_S3_REGION_NAME=
-USE_S3=
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123
+AWS_STORAGE_BUCKET_NAME=jandig-cdn
+AWS_PRIVATE_STORAGE_BUCKET_NAME=jandig-private-cdn
+AWS_S3_REGION_NAME=us-east-2
+AWS_STATIC_LOCATION=static
+MINIO_S3_ENDPOINT_URL=http://storage:9000
+MINIO_SITE_REGION=us-east-2
+MINIO_USER_ACCESS_KEY=minio-access-key
+MINIO_USER_SECRET_KEY=minio-secret-key
 
 ## Postgres variables
 POSTGRES_HOST=postgres

src/ARte/config/settings.py

@@ -39,20 +39,28 @@
 # Application definition
 
 
+def traces_sampler(sampling_context):
+    url = sampling_context["wsgi_environ"]["PATH_INFO"]
+    if "/status" in url:
+        return 0
+    elif "/static/" in url:
+        return 0
+    return 0.1
+
 sentry_sdk.init(
     dsn="https://081a2c3476b24a9f9a51d74bde539b62@o968990.ingest.sentry.io/5920229",
     integrations=[DjangoIntegration()],
-
-    # Set traces_sample_rate to 1.0 to capture 100%
-    # of transactions for performance monitoring.
-    # We recommend adjusting this value in production.
-    traces_sample_rate=1.0,
-
     # If you wish to associate users to errors (assuming you are using
     # django.contrib.auth) you may enable sending PII data.
-    send_default_pii=True
+    send_default_pii=True,
+    traces_sampler=traces_sampler,
 )
 
+# Sentry configuration
+ENABLE_SENTRY_LOGS = env("ENABLE_SENTRY_LOGS", default=False)
+HEALTH_CHECK_URL = env("HEALTH_CHECK_URL", default="api/v1/status/")
+
+
 INSTALLED_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -184,41 +192,53 @@ def debug(request):
 USE_TZ = True
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/2.1/howto/static-files/
 
-COLLECT_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
-USE_S3 = os.getenv("USE_S3", "False").lower() == "true"
+# AWS credentials
+AWS_S3_OBJECT_PARAMETERS = {
+    "CacheControl": "max-age=86400",
+}
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "")
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "")
+AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME", "us-east-2")
+AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL", None)
+AWS_STATIC_LOCATION = os.getenv("AWS_STATIC_LOCATION", "static")
+AWS_MEDIA_LOCATION = os.getenv("AWS_MEDIA_LOCATION", "media")
+USE_MINIO = os.getenv("USE_MINIO", "false").lower() in ("true", "True", "1")
+if USE_MINIO:
+    AWS_S3_ENDPOINT_URL = os.getenv("MINIO_S3_ENDPOINT_URL", "http://storage:9000")
+    AWS_S3_CUSTOM_DOMAIN = f"localhost:9000/{AWS_STORAGE_BUCKET_NAME}"
+    AWS_S3_USE_SSL = False
+    AWS_S3_SECURE_URLS = False
+    HTTP_PROTOCOL = "http"
+else:
+    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+    HTTP_PROTOCOL = "https"
+
+# Static configuration
+# Add your own apps statics in this list
 STATICFILES_DIRS = [
     os.path.join(BASE_DIR, 'core', 'static'),
     os.path.join(BASE_DIR, 'users', 'static')
 ]
-if USE_S3:
-    # AWS credentials
-    AWS_S3_OBJECT_PARAMETERS = {
-        "CacheControl": "max-age=86400",
-    }
-    AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "")
-    AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "")
-    AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "")
-    AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME", "us-east-2")
-    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
-    AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL", None)
-    AWS_STATIC_LOCATION = os.getenv("AWS_STATIC_LOCATION", "static")
+COLLECT_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
+STATIC_ROOT = os.path.join(COLLECT_DIR, 'collect')
+STATIC_URL = f"{HTTP_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{AWS_STATIC_LOCATION}/"
+STATICFILES_STORAGE = "config.storage_backends.StaticStorage"
 
-    # Static configuration
-    STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_STATIC_LOCATION}/"
-    STATICFILES_STORAGE = "config.storage_backends.StaticStorage"
+MEDIA_ROOT = os.path.join(BASE_DIR, 'users', 'media')
+MEDIA_URL = f"{HTTP_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{AWS_MEDIA_LOCATION}/"
 
-    AWS_PUBLIC_MEDIA_LOCATION = "media/public"
-    DEFAULT_FILE_STORAGE = "config.storage_backends.PublicMediaStorage"
-else:
-    STATIC_URL = '/static/'
-    STATIC_ROOT = os.path.join(COLLECT_DIR, 'collect')
-
+AWS_PUBLIC_MEDIA_LOCATION = "media/public"
+DEFAULT_FILE_STORAGE = "config.storage_backends.PublicMediaStorage"
+
+AWS_PRIVATE_MEDIA_LOCATION = "media/private"
+PRIVATE_FILE_STORAGE = "config.storage_backends.PrivateMediaStorage"
+
+AWS_PRIVATE_MEDIA_DIFFERENT_BUCKET_LOCATION = "media/private"
+AWS_PRIVATE_STORAGE_BUCKET_NAME = os.getenv("AWS_PRIVATE_STORAGE_BUCKET_NAME", "")
+PRIVATE_FILE_DIFFERENT_BUCKET_STORAGE = "config.storage_backends.PrivateMediaStorage"
 
-MEDIA_URL = '/media/'
-MEDIA_ROOT = os.path.join(BASE_DIR, 'users', 'media')
 
 # LOGIN / LOGOUT
 LOGIN_URL = 'login'

src/ARte/core/helpers.py

@@ -1,7 +1,21 @@
 from django.contrib.staticfiles.templatetags.staticfiles import static
 
+from django.core.files.storage import default_storage
+
 def handle_upload_image(image):
     path = "core" + static(f"images/{image.name}")
     with open(path, 'wb+') as destination:
         for chunk in image.chunks():
             destination.write(chunk)
+
+def handle_upload_patt(patt):
+    path = "core" + static(f"patts/{patt.name}")
+    with default_storage.open(path, 'wb') as destination:
+        for chunk in patt.chunks():
+            destination.write(chunk)
+
+def handle_upload_marker(image):
+    path = "core" + static(f"markers/{image.name}")
+    with open(path, 'wb+') as destination:
+        for chunk in image.chunks():
+            destination.write(chunk)

src/ARte/core/models.py

@@ -1,9 +1,36 @@
 from django.db import models
+from config.storage_backends import PublicMediaStorage
+from .helpers import handle_upload_marker, handle_upload_patt
 from users.models import Profile
 from django.dispatch import receiver
 from django.db.models.signals import post_delete
 import re
 
+from django.core.files import File
+from django.core.files.base import ContentFile
+from io import BytesIO, StringIO
+from PIL import Image
+from pymarker.core import generate_marker_from_image, generate_patt_from_image
+
+def create_patt(filename, original_filename):
+    filestorage = PublicMediaStorage()
+    with Image.open(filestorage.open(filename)) as image:
+        patt_str = generate_patt_from_image(image)
+        # string_file = StringIO(patt_str.encode('UTF-8'))
+        # string_file.name = original_filename
+        patt_file = filestorage.save("patts/" + original_filename + ".patt", ContentFile(patt_str.encode('utf-8')))
+        return patt_file
+
+
+def create_marker(filename, original_filename):
+    filestorage = PublicMediaStorage()
+    with Image.open(filestorage.open(filename)) as image:
+        marker_image = generate_marker_from_image(image)
+        marker_image.name = original_filename
+        marker_image.__commited = False
+        # marker = filestorage.save("markers/" + original_filename, marker_image)
+        return marker_image
+
 class Marker(models.Model):
     owner = models.ForeignKey(Profile, on_delete=models.DO_NOTHING)
     source = models.ImageField(upload_to='markers/')
@@ -12,6 +39,25 @@ class Marker(models.Model):
     title = models.CharField(max_length=60, default='')
     patt = models.FileField(upload_to='patts/')
 
+    def save(self, *args, **kwargs):
+        # filestorage = PublicMediaStorage()
+        # # Image Filename
+        # original_filename = self.source.name
+        # filename = filestorage.save(f"original_{original_filename}", self.source)
+        # # Complete Image URL on storage
+        # print("aaaaa"*30)
+        # with Image.open(self.source) as image:
+        #     print(image)
+        # print("aaaaa"*30)
+        # # fileurl = filestorage.url(filename)
+        # print(filename)
+        # self.source = create_marker(filename, original_filename)
+        # self.patt = create_patt(filename, original_filename)
+        print("B"*30)
+        print(self.source)
+        print(self.patt)
+        print("B"*30)
+        super().save(*args, **kwargs)
 
     def __str__(self):
         return self.source.name

src/ARte/core/urls.py

@@ -17,4 +17,5 @@
     path('i18n/', include('django.conf.urls.i18n')),
     path('see_all/', see_all, name='see-all'),
     path('robots.txt/', robots_txt),
+    path('status/', health_check),
 ]

src/ARte/core/views_s/home.py

@@ -14,3 +14,7 @@ def community(request):
 
 def marker_generator(request):
     return render(request,'core/generator.html',{})
+
+def health_check(request):
+    return JsonResponse({'status': 'ok'}, status=200)
+

src/ARte/users/forms.py

@@ -170,6 +170,12 @@ class RecoverPasswordCodeForm(forms.Form):
     verification_code = forms.CharField(label='Verification code', max_length="200")
 
 
+from django.core.files.base import ContentFile, File
+from io import BytesIO, StringIO
+from PIL import Image
+from django.core.files.images import ImageFile
+from pymarker.core import generate_marker_from_image, generate_patt_from_image
+
 class UploadMarkerForm(forms.ModelForm):
 
     def __init__(self, *args, **kwargs):
@@ -178,16 +184,29 @@ def __init__(self, *args, **kwargs):
         log.warning(self.fields)
         self.fields['source'].widget.attrs['placeholder'] = _('browse file')
         self.fields['source'].widget.attrs['accept'] = 'image/png, image/jpg'
-        self.fields['patt'].widget.attrs['placeholder'] = _('browse file')
-        self.fields['patt'].widget.attrs['accept'] = '.patt'
         self.fields['author'].widget.attrs['placeholder'] = _('declare different author name')
         self.fields['title'].widget.attrs['placeholder'] = _("Marker's title")
 
     class Meta:
         model = Marker
-        fields = ('source', 'author', 'title', 'patt')
-
-
+        exclude = ('owner', 'uploaded_at', 'patt')
+
+    def save(self, *args, **kwargs):
+        with Image.open(self.instance.source) as image:
+            pil_image = generate_marker_from_image(image)
+            blob = BytesIO()
+            pil_image.save(blob, 'JPEG')
+            filename = self.instance.source.name
+            self.instance.source.save(filename, File(blob),save=False)
+            patt_str = generate_patt_from_image(image)
+            self.instance.patt.save(filename + ".patt", ContentFile(patt_str.encode('utf-8')),save=False)
+
+            if kwargs.get("owner"):
+                self.instance.owner = kwargs.get("owner")
+                del kwargs["owner"]
+            self.instance.save()
+
+            return super(UploadMarkerForm,self).save(*args, **kwargs)
 
 class UploadObjectForm(forms.ModelForm):