[Fixes #12124] GNIP 100: Assets

.env.sample

@@ -168,6 +168,7 @@ SECRET_KEY='{secret_key}'
 
 STATIC_ROOT=/mnt/volumes/statics/static/
 MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+ASSETS_ROOT=/mnt/volumes/statics/assets/
 GEOIP_PATH=/mnt/volumes/statics/geoip.db
 
 CACHE_BUSTING_STATIC_ENABLED=False

geonode/assets/apps.py

@@ -0,0 +1,35 @@
+#########################################################################
+#
+# Copyright (C) 2016 OSGeo
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+from django.apps import AppConfig
+
+from geonode.notifications_helper import NotificationsAppConfigBase
+
+
+class BaseAppConfig(NotificationsAppConfigBase, AppConfig):
+    name = "geonode.assets"
+
+    def ready(self):
+        super().ready()
+        run_setup_hooks()
+
+
+def run_setup_hooks(*args, **kwargs):
+    from geonode.assets.handlers import asset_handler_registry
+
+    asset_handler_registry.init_registry()

geonode/assets/handlers.py

@@ -0,0 +1,91 @@
+import logging
+
+from django.conf import settings
+from django.http import HttpResponse
+from django.utils.module_loading import import_string
+
+from geonode.assets.models import Asset
+
+logger = logging.getLogger(__name__)
+
+
+class AssetHandlerInterface:
+
+    def handled_asset_class(self):
+        raise NotImplementedError()
+
+    def create(self, title, description, type, owner, *args, **kwargs):
+        raise NotImplementedError()
+
+    def remove_data(self, asset: Asset, **kwargs):
+        raise NotImplementedError()
+
+    def replace_data(self, asset: Asset, files: list):
+        raise NotImplementedError()
+
+    def clone(self, asset: Asset) -> Asset:
+        """
+        Creates a copy in the DB and copies the underlying data as well
+        """
+        raise NotImplementedError()
+
+    def create_link_url(self, asset: Asset) -> str:
+        raise NotImplementedError()
+
+    def get_download_handler(self, asset: Asset):
+        raise NotImplementedError()
+
+    def get_storage_manager(self, asset):
+        raise NotImplementedError()
+
+
+class AssetDownloadHandlerInterface:
+
+    def create_response(self, asset: Asset, attachment: bool = False, basename=None) -> HttpResponse:
+        raise NotImplementedError()
+
+
+class AssetHandlerRegistry:
+    _registry = {}
+    _default_handler = None
+
+    def init_registry(self):
+        self.register_asset_handlers()
+        self.set_default_handler()
+
+    def register_asset_handlers(self):
+        for module_path in settings.ASSET_HANDLERS:
+            handler = import_string(module_path)
+            self.register(handler)
+        logger.info(f"Registered Asset handlers: {', '.join(settings.ASSET_HANDLERS)}")
+
+    def set_default_handler(self):
+        # check if declared class is registered
+        for handler in self._registry.values():
+            if ".".join([handler.__class__.__module__, handler.__class__.__name__]) == settings.DEFAULT_ASSET_HANDLER:
+                self._default_handler = handler
+                break
+
+        if self._default_handler is None:
+            logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}")
+        else:
+            logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}")
+
+    def register(self, asset_handler_class):
+        self._registry[asset_handler_class.handled_asset_class()] = asset_handler_class()
+
+    def get_default_handler(self) -> AssetHandlerInterface:
+        return self._default_handler
+
+    def get_handler(self, asset):
+        asset_cls = asset if isinstance(asset, type) else asset.__class__
+        ret = self._registry.get(asset_cls, None)
+        if not ret:
+            logger.warning(f"Could not find asset handler for {asset_cls}::{asset.__class__}")
+            logger.warning("Available asset types:")
+            for k, v in self._registry.items():
+                logger.warning(f"{k} --> {v.__class__.__name__}")
+        return ret
+
+
+asset_handler_registry = AssetHandlerRegistry()

geonode/assets/local.py

@@ -0,0 +1,159 @@
+import datetime
+import logging
+import os
+
+from django.conf import settings
+from django.http import HttpResponse
+from django.urls import reverse
+from django_downloadview import DownloadResponse
+
+from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface
+from geonode.assets.models import LocalAsset
+from geonode.storage.manager import DefaultStorageManager, StorageManager
+from geonode.utils import build_absolute_uri, mkdtemp
+
+logger = logging.getLogger(__name__)
+
+_asset_storage_manager = StorageManager(
+    concrete_storage_manager=DefaultStorageManager(location=os.path.dirname(settings.ASSETS_ROOT))
+)
+
+
+class LocalAssetHandler(AssetHandlerInterface):
+    @staticmethod
+    def handled_asset_class():
+        return LocalAsset
+
+    def get_download_handler(self, asset):
+        return LocalAssetDownloadHandler()
+
+    def get_storage_manager(self, asset):
+        return _asset_storage_manager
+
+    def _create_asset_dir(self):
+        return os.path.normpath(
+            mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
+        )
+
+    def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs):
+        if not files:
+            raise ValueError("File(s) expected")
+
+        if clone_files:
+            prefix = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+            files = _asset_storage_manager.copy_files_list(files, dir=settings.ASSETS_ROOT, dir_prefix=prefix)
+            # TODO: please note the copy_files_list will make flat any directory structure
+
+        asset = LocalAsset(
+            title=title,
+            description=description,
+            type=type,
+            owner=owner,
+            created=datetime.datetime.now(),
+            location=files,
+        )
+        asset.save()
+        return asset
+
+    def remove_data(self, asset: LocalAsset):
+        """
+        Removes the files related to an Asset.
+        Only files within the Assets directory are removed
+        """
+        removed_dir = set()
+        for file in asset.location:
+            is_managed = self._is_file_managed(file)
+            if is_managed:
+                logger.info(f"Removing asset file {file}")
+                _asset_storage_manager.delete(file)
+                removed_dir.add(os.path.dirname(file))
+            else:
+                logger.info(f"Not removing asset file outside asset directory {file}")
+
+        # TODO: in case of subdirs, make sure that all the tree is removed in the proper order
+        for dir in removed_dir:
+            if not os.path.exists(dir):
+                logger.warning(f"Trying to remove not existing asset directory {dir}")
+                continue
+            if not os.listdir(dir):
+                logger.info(f"Removing empty asset directory {dir}")
+                os.rmdir(dir)
+
+    def replace_data(self, asset: LocalAsset, files: list):
+        self.remove_data(asset)
+        asset.location = files
+        asset.save()
+
+    def clone(self, source: LocalAsset) -> LocalAsset:
+        # get a new asset instance to be edited and stored back
+        asset = LocalAsset.objects.get(pk=source.pk)
+        # only copy files if they are managed
+        if self._are_files_managed(asset.location):
+            asset.location = _asset_storage_manager.copy_files_list(
+                asset.location, dir=settings.ASSETS_ROOT, dir_prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+            )
+        # it's a polymorphic object, we need to null both IDs
+        # https://django-polymorphic.readthedocs.io/en/stable/advanced.html#copying-polymorphic-objects
+        asset.pk = None
+        asset.id = None
+        asset.save()
+        asset.refresh_from_db()
+        return asset
+
+    def create_download_url(self, asset) -> str:
+        return build_absolute_uri(reverse("assets-download", args=(asset.pk,)))
+
+    def create_link_url(self, asset) -> str:
+        return build_absolute_uri(reverse("assets-link", args=(asset.pk,)))
+
+    def _is_file_managed(self, file) -> bool:
+        assets_root = os.path.normpath(settings.ASSETS_ROOT)
+        return file.startswith(assets_root)
+
+    def _are_files_managed(self, files: list) -> bool:
+        """
+        :param files: files to be checked
+        :return: True if all files are managed, False is no file is managed
+        :raise: ValueError if both managed and unmanaged files are in the list
+        """
+        managed = unmanaged = None
+        for file in files:
+            if self._is_file_managed(file):
+                managed = True
+            else:
+                unmanaged = True
+            if managed and unmanaged:
+                logger.error(f"Both managed and unmanaged files are present: {files}")
+                raise ValueError("Both managed and unmanaged files are present")
+
+        return bool(managed)
+
+
+class LocalAssetDownloadHandler(AssetDownloadHandlerInterface):
+
+    def create_response(self, asset: LocalAsset, attachment: bool = False, basename=None) -> HttpResponse:
+        if not asset.location:
+            return HttpResponse("Asset does not contain any data", status=500)
+
+        if len(asset.location) > 1:
+            logger.warning("TODO: Asset contains more than one file. Download needs to be implemented")
+
+        file0 = asset.location[0]
+        filename = os.path.basename(file0)
+        orig_base, ext = os.path.splitext(filename)
+        outname = f"{basename or orig_base}{ext}"
+
+        if _asset_storage_manager.exists(file0):
+            logger.info(f"Returning file {file0} with name {outname}")
+
+            return DownloadResponse(
+                _asset_storage_manager.open(file0).file,
+                basename=f"{outname}",
+                attachment=attachment,
+            )
+        else:
+            logger.warning(f"Internal file {file0} not found for asset  {asset.id}")
+            return HttpResponse(f"Internal file not found for asset {asset.id}", status=500)
+
+
+asset_handler_registry.register(LocalAssetHandler)

geonode/assets/migrations/0001_initial.py

@@ -0,0 +1,63 @@
+# Generated by Django 4.2.9 on 2024-04-24 10:02
+
+from django.conf import settings
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("base", "0091_alter_hierarchicalkeyword_slug"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Asset",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("title", models.CharField(max_length=255)),
+                ("description", models.TextField(blank=True, null=True)),
+                ("type", models.CharField(max_length=255)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                (
+                    "polymorphic_ctype",
+                    models.ForeignKey(
+                        editable=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="polymorphic_%(app_label)s.%(class)s_set+",
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name_plural": "Assets",
+            },
+        ),
+        migrations.CreateModel(
+            name="LocalAsset",
+            fields=[
+                (
+                    "asset_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="assets.asset",
+                    ),
+                ),
+                ("location", models.JSONField(blank=True, default=list)),
+            ],
+            options={
+                "verbose_name_plural": "Local assets",
+            },
+            bases=("assets.asset",),
+        ),
+    ]