diff --git a/.env.sample b/.env.sample index 2ec4a7b8c9a..d83a7bfdbf1 100644 --- a/.env.sample +++ b/.env.sample @@ -168,6 +168,7 @@ SECRET_KEY='{secret_key}' STATIC_ROOT=/mnt/volumes/statics/static/ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +ASSETS_ROOT=/mnt/volumes/statics/assets/ GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False diff --git a/.gitignore b/.gitignore index 7a455624878..2080d18191d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ geonode/local_settings.py # Uploaded files geonode/uploaded +geonode/assets_data #Testing output .coverage diff --git a/geonode/assets/__init__.py b/geonode/assets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/admin.py b/geonode/assets/admin.py new file mode 100644 index 00000000000..f9f1e9fcdf1 --- /dev/null +++ b/geonode/assets/admin.py @@ -0,0 +1,55 @@ +import json +import logging +from django.db import models +from django.forms import widgets +from django.contrib import admin + +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import LocalAsset +from geonode.base.models import Link + +logger = logging.getLogger(__name__) + + +class PrettyJSONWidget(widgets.Textarea): + + def format_value(self, value): + try: + value = json.dumps(json.loads(value), indent=2, sort_keys=True) + # these lines will try to adjust size of TextArea to fit to content + row_lengths = [len(r) for r in value.split("\n")] + self.attrs["rows"] = min(max(len(row_lengths) + 2, 10), 30) + self.attrs["cols"] = min(max(max(row_lengths) + 2, 40), 120) + return value + except Exception as e: + logger.warning("Error while formatting JSON: {}".format(e)) + return super(PrettyJSONWidget, self).format_value(value) + + +@admin.register(LocalAsset) +class LocalAssetAdmin(admin.ModelAdmin): + model = LocalAsset + + list_display = ("id", "title", "type", "owner", "created_formatted", "managed", "links", "link0") + list_display_links = ("id", "title") + + formfield_overrides = {models.JSONField: {"widget": PrettyJSONWidget}} + + def created_formatted(self, obj): + return obj.created.strftime("%Y-%m-%d %H:%M:%S") + + def links(self, obj): + return Link.objects.filter(asset=obj).count() + + def link0(self, obj): + link = Link.objects.filter(asset=obj).first() + return f"{link.link_type} {link.extension}: {link.name}" if link else None + + def managed(self, obj) -> bool: + try: + return LocalAssetHandler._is_file_managed(obj.location[0]) + except Exception as e: + logger.error(f"Bad location for asset obj: {e}") + return None + + managed.boolean = True diff --git a/geonode/assets/apps.py b/geonode/assets/apps.py new file mode 100644 index 00000000000..c994f7bdb09 --- /dev/null +++ b/geonode/assets/apps.py @@ -0,0 +1,35 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + +from geonode.notifications_helper import NotificationsAppConfigBase + + +class BaseAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.assets" + + def ready(self): + super().ready() + run_setup_hooks() + + +def run_setup_hooks(*args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.init_registry() diff --git a/geonode/assets/handlers.py b/geonode/assets/handlers.py new file mode 100644 index 00000000000..3df019c521c --- /dev/null +++ b/geonode/assets/handlers.py @@ -0,0 +1,91 @@ +import logging + +from django.conf import settings +from django.http import HttpResponse +from django.utils.module_loading import import_string + +from geonode.assets.models import Asset + +logger = logging.getLogger(__name__) + + +class AssetDownloadHandlerInterface: + + def create_response(self, asset: Asset, attachment: bool = False, basename=None, path=None) -> HttpResponse: + raise NotImplementedError() + + +class AssetHandlerInterface: + + def handled_asset_class(self): + raise NotImplementedError() + + def create(self, title, description, type, owner, *args, **kwargs): + raise NotImplementedError() + + def remove_data(self, asset: Asset, **kwargs): + raise NotImplementedError() + + def replace_data(self, asset: Asset, files: list): + raise NotImplementedError() + + def clone(self, asset: Asset) -> Asset: + """ + Creates a copy in the DB and copies the underlying data as well + """ + raise NotImplementedError() + + def create_link_url(self, asset: Asset) -> str: + raise NotImplementedError() + + def get_download_handler(self, asset: Asset = None) -> AssetDownloadHandlerInterface: + raise NotImplementedError() + + def get_storage_manager(self, asset=None): + raise NotImplementedError() + + +class AssetHandlerRegistry: + _registry = {} + _default_handler = None + + def init_registry(self): + self.register_asset_handlers() + self.set_default_handler() + + def register_asset_handlers(self): + for module_path in settings.ASSET_HANDLERS: + handler = import_string(module_path) + self.register(handler) + logger.info(f"Registered Asset handlers: {', '.join(settings.ASSET_HANDLERS)}") + + def set_default_handler(self): + # check if declared class is registered + for handler in self._registry.values(): + if ".".join([handler.__class__.__module__, handler.__class__.__name__]) == settings.DEFAULT_ASSET_HANDLER: + self._default_handler = handler + break + + if self._default_handler is None: + logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}") + else: + logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}") + + def register(self, asset_handler_class): + self._registry[asset_handler_class.handled_asset_class()] = asset_handler_class() + + def get_default_handler(self) -> AssetHandlerInterface: + return self._default_handler + + def get_handler(self, asset): + asset_cls = asset if isinstance(asset, type) else asset.__class__ + ret = self._registry.get(asset_cls, None) + if not ret: + logger.warning(f"Could not find asset handler for {asset_cls}::{asset.__class__}") + logger.warning("Available asset types:") + for k, v in self._registry.items(): + logger.warning(f"{k} --> {v.__class__.__name__}") + return ret + + +asset_handler_registry = AssetHandlerRegistry() diff --git a/geonode/assets/local.py b/geonode/assets/local.py new file mode 100644 index 00000000000..0d0c3b71aea --- /dev/null +++ b/geonode/assets/local.py @@ -0,0 +1,257 @@ +import datetime +import logging +import os +import shutil + +from django.conf import settings +from django.http import HttpResponse +from django.urls import reverse +from django_downloadview import DownloadResponse + +from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface +from geonode.assets.models import LocalAsset +from geonode.storage.manager import DefaultStorageManager, StorageManager +from geonode.utils import build_absolute_uri, mkdtemp + +logger = logging.getLogger(__name__) + +_asset_storage_manager = StorageManager( + concrete_storage_manager=DefaultStorageManager(location=os.path.dirname(settings.ASSETS_ROOT)) +) + + +class DefaultLocalLinkUrlHandler: + def get_link_url(self, asset: LocalAsset): + return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + + +class IndexLocalLinkUrlHandler: + def get_link_url(self, asset: LocalAsset): + return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + f"/{os.path.basename(asset.location[0])}" + + +class LocalAssetHandler(AssetHandlerInterface): + + link_url_handlers = {"3dtiles": IndexLocalLinkUrlHandler()} + + @staticmethod + def handled_asset_class(): + return LocalAsset + + def get_download_handler(self, asset=None): + return LocalAssetDownloadHandler() + + def get_storage_manager(self, asset): + return _asset_storage_manager + + def get_link_url_handler(self, asset): + return self.link_url_handlers.get(asset.type, None) or DefaultLocalLinkUrlHandler() + + def _create_asset_dir(self): + return os.path.normpath( + mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + ) + + def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs): + if not files: + raise ValueError("File(s) expected") + + if clone_files: + files = self._copy_data(files) + + asset = LocalAsset( + title=title, + description=description, + type=type, + owner=owner, + created=datetime.datetime.now(), + location=files, + ) + asset.save() + return asset + + def remove_data(self, asset: LocalAsset): + """ + Removes the files related to an Asset. + Only files within the Assets directory are removed + """ + if self._are_files_managed(asset): + logger.info(f"Removing files for asset {asset.pk}") + base = self._get_managed_dir(asset) + logger.info(f"Removing asset path {base} for asset {asset.pk}") + shutil.rmtree(base) + else: + logger.info(f"Not removing unmanaged files for asset {asset.pk}") + + def replace_data(self, asset: LocalAsset, files: list): + self.remove_data(asset) + asset.location = files + asset.save() + + def _copy_data(self, files): + new_path = self._create_asset_dir() + logger.info(f"Copying asset data from {files} into {new_path}") + new_files = [] + for file in files: + if os.path.isdir(file): + dst = os.path.join(new_path, os.path.basename(file)) + logging.info(f"Copying into {dst} directory {file}") + new_dir = shutil.copytree(file, dst) + new_files.append(new_dir) + elif os.path.isfile(file): + logging.info(f"Copying into {new_path} file {os.path.basename(file)}") + new_file = shutil.copy2(file, new_path) + new_files.append(new_file) + else: + logger.warning(f"Not copying path {file}") + + return new_files + + def _clone_data(self, source_dir): + new_path = self._create_asset_dir() + logger.info(f"Cloning asset data from {source_dir} into {new_path}") + + if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: + # value is always set by default as None + # https://docs.djangoproject.com/en/3.2/ref/settings/#file-upload-directory-permissions + os.chmod(new_path, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + + shutil.copytree(source_dir, new_path, dirs_exist_ok=True) + + return new_path + + def clone(self, source: LocalAsset) -> LocalAsset: + # get a new asset instance to be edited and stored back + asset = LocalAsset.objects.get(pk=source.pk) + + # only copy files if they are managed + if self._are_files_managed(asset): + base = self._get_managed_dir(asset) + cloned = self._clone_data(base) + asset.location = [os.path.normpath(file).replace(base, cloned) for file in asset.location] + + # it's a polymorphic object, we need to null both IDs + # https://django-polymorphic.readthedocs.io/en/stable/advanced.html#copying-polymorphic-objects + asset.pk = None + asset.id = None + asset.save() + asset.refresh_from_db() + return asset + + def create_download_url(self, asset) -> str: + return build_absolute_uri(reverse("assets-download", args=(asset.pk,))) + + def create_link_url(self, asset) -> str: + return self.get_link_url_handler(asset).get_link_url(asset) + + @classmethod + def _is_file_managed(cls, file) -> bool: + assets_root = os.path.normpath(settings.ASSETS_ROOT) + return file.startswith(assets_root) + + @classmethod + def _are_files_managed(cls, asset: LocalAsset) -> bool: + """ + :param files: files to be checked + :return: True if all files are managed, False is no file is managed + :raise: ValueError if both managed and unmanaged files are in the list + """ + managed = unmanaged = None + for file in asset.location: + if cls._is_file_managed(file): + managed = True + else: + unmanaged = True + if managed and unmanaged: + logger.error(f"Both managed and unmanaged files are present on Asset {asset.pk}: {asset.location}") + raise ValueError("Both managed and unmanaged files are present") + + return bool(managed) + + @classmethod + def _get_managed_dir(cls, asset): + if not asset.location: + raise ValueError("Asset does not have any associated file") + + assets_root = os.path.normpath(settings.ASSETS_ROOT) + base_common = None + + for file in asset.location: + if not cls._is_file_managed(file): + raise ValueError("Asset is unmanaged") + + norm_file = os.path.normpath(file) + relative = norm_file.removeprefix(assets_root) + base = os.path.split(relative)[0].lstrip("/") + + if base_common: + if base_common != base: + raise ValueError(f"Mismatching base dir in asset files - Asset {asset.pk}") + else: + base_common = base + + managed_dir = os.path.join(assets_root, base_common) + + if not os.path.exists(managed_dir): + raise ValueError(f"Common dir '{managed_dir}' does not exist - Asset {asset.pk}") + + if not os.path.isdir(managed_dir): + raise ValueError(f"Common dir '{managed_dir}' does not seem to be a directory - Asset {asset.pk}") + + if assets_root == managed_dir: # dunno if this can ever happen, but better safe than sorry + raise ValueError(f"Common dir '{managed_dir}' matches the whole Assets dir - Asset {asset.pk}") + + return managed_dir + + +class LocalAssetDownloadHandler(AssetDownloadHandlerInterface): + + def create_response( + self, asset: LocalAsset, attachment: bool = False, basename: str = None, path: str = None + ) -> HttpResponse: + if not asset.location: + return HttpResponse("Asset does not contain any data", status=500) + + if len(asset.location) > 1: + logger.warning("TODO: Asset contains more than one file. Download needs to be implemented") + + file0 = asset.location[0] + if not path: # use the file definition + if not os.path.isfile(file0): + logger.warning(f"Default file {file0} not found for asset {asset.id}") + return HttpResponse(f"Default file not found for asset {asset.id}", status=400) + localfile = file0 + + else: # a specific file is requested + if "/../" in path: # we may want to improve fraudolent request detection + logger.warning(f"Tentative path traversal for asset {asset.id}") + return HttpResponse(f"File not found for asset {asset.id}", status=400) + + if os.path.isfile(file0): + dir0 = os.path.dirname(file0) + elif os.path.isdir(file0): + dir0 = file0 + else: + return HttpResponse(f"Unexpected internal location '{file0}' for asset {asset.id}", status=500) + + localfile = os.path.join(dir0, path) + logger.debug(f"Requested path {dir0} + {path}") + + if os.path.isfile(localfile): + filename = os.path.basename(localfile) + orig_base, ext = os.path.splitext(filename) + outname = f"{basename or orig_base or 'file'}{ext}" + + logger.info(f"Returning file '{localfile}' with name '{outname}'") + + return DownloadResponse( + _asset_storage_manager.open(localfile).file, + basename=f"{outname}", + attachment=attachment, + ) + else: + logger.warning(f"Internal file {localfile} not found for asset {asset.id}") + return HttpResponse(f"Internal file not found for asset {asset.id}", status=404 if path else 500) + + +asset_handler_registry.register(LocalAssetHandler) diff --git a/geonode/assets/migrations/0001_initial.py b/geonode/assets/migrations/0001_initial.py new file mode 100644 index 00000000000..8a1ef0849a4 --- /dev/null +++ b/geonode/assets/migrations/0001_initial.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:02 + +from django.conf import settings +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("base", "0091_alter_hierarchicalkeyword_slug"), + ] + + operations = [ + migrations.CreateModel( + name="Asset", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("title", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, null=True)), + ("type", models.CharField(max_length=255)), + ("created", models.DateTimeField(auto_now_add=True)), + ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "polymorphic_ctype", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polymorphic_%(app_label)s.%(class)s_set+", + to="contenttypes.contenttype", + ), + ), + ], + options={ + "verbose_name_plural": "Assets", + }, + ), + migrations.CreateModel( + name="LocalAsset", + fields=[ + ( + "asset_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="assets.asset", + ), + ), + ("location", models.JSONField(blank=True, default=list)), + ], + options={ + "verbose_name_plural": "Local assets", + }, + bases=("assets.asset",), + ), + ] diff --git a/geonode/assets/migrations/__init__.py b/geonode/assets/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/models.py b/geonode/assets/models.py new file mode 100644 index 00000000000..a4fe3a26de6 --- /dev/null +++ b/geonode/assets/models.py @@ -0,0 +1,48 @@ +from django.db import models +from polymorphic.managers import PolymorphicManager +from polymorphic.models import PolymorphicModel +from django.db.models import signals +from django.contrib.auth import get_user_model + + +class Asset(PolymorphicModel): + """ + A generic data linked to a ResourceBase + """ + + title = models.CharField(max_length=255, null=False, blank=False) + description = models.TextField(null=True, blank=True) + type = models.CharField(max_length=255, null=False, blank=False) + owner = models.ForeignKey(get_user_model(), null=False, blank=False, on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + + objects = PolymorphicManager() + + class Meta: + verbose_name_plural = "Assets" + + def __str__(self) -> str: + return super().__str__() + + +class LocalAsset(Asset): + """ + Local resource, will replace the files + """ + + location = models.JSONField(default=list, blank=True) + + class Meta: + verbose_name_plural = "Local assets" + + def __str__(self) -> str: + return f"{self.__class__.__name__}: {self.type}|{self.title}" + + +def cleanup_asset_data(instance, *args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.get_handler(instance).remove_data(instance) + + +signals.post_delete.connect(cleanup_asset_data, sender=LocalAsset) diff --git a/geonode/assets/serializers.py b/geonode/assets/serializers.py new file mode 100644 index 00000000000..999cf8005e7 --- /dev/null +++ b/geonode/assets/serializers.py @@ -0,0 +1,78 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.contrib.auth import get_user_model + +from dynamic_rest.serializers import DynamicModelSerializer +from dynamic_rest.fields.fields import DynamicComputedField + +from geonode.assets.models import ( + Asset, + LocalAsset, +) + +logger = logging.getLogger(__name__) + + +class ClassTypeField(DynamicComputedField): + + def get_attribute(self, instance): + return type(instance).__name__ + + +class SimpleUserSerializer(DynamicModelSerializer): + class Meta: + model = get_user_model() + name = "user" + fields = ("pk", "username") + + +class AssetSubclassField(DynamicComputedField): + """ + Just an ugly hack. + TODO: We need a way to automatically use a proper serializer for each Asset subclass + in order to render different instances in a list + """ + + def get_attribute(self, instance): + if type(instance).__name__ == "LocalAsset": + return {"locations": instance.location} + + return None + + +class AssetSerializer(DynamicModelSerializer): + + owner = SimpleUserSerializer(embed=False) + asset_type = ClassTypeField() + subinfo = AssetSubclassField() + + class Meta: + model = Asset + name = "asset" + # fields = ("pk", "title", "description", "type", "owner", "created") + fields = ("pk", "title", "description", "type", "owner", "created", "asset_type", "subinfo") + + +class LocalAssetSerializer(AssetSerializer): + class Meta(AssetSerializer.Meta): + model = LocalAsset + name = "local_asset" + fields = AssetSerializer.Meta.fields + ("location",) diff --git a/geonode/assets/tests.py b/geonode/assets/tests.py new file mode 100644 index 00000000000..75fecd1d7e2 --- /dev/null +++ b/geonode/assets/tests.py @@ -0,0 +1,260 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import logging +import shutil +import io +import json + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.urls import reverse + +from rest_framework.test import APITestCase + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import Asset, LocalAsset + +logger = logging.getLogger(__name__) + +ONE_JSON = os.path.join(os.path.dirname(__file__), "tests/data/one.json") +TWO_JSON = os.path.join(os.path.dirname(__file__), "tests/data/two.json") +THREE_JSON = os.path.join(os.path.dirname(__file__), "tests/data/three.json") + + +class AssetsTests(APITestCase): + + def test_handler_registry(self): + # Test registry + self.assertIsNotNone(asset_handler_registry) + # Test default handler + asset_handler = asset_handler_registry.get_default_handler() + self.assertIsNotNone(asset_handler) + self.assertIsInstance(asset_handler, LocalAssetHandler, "Bad default Asset handler found") + # Test None + self.assertIsNone(asset_handler_registry.get_handler(None)) + # Test class without handler + self.assertIsNone(asset_handler_registry.get_handler(AssetsTests)) + + def test_creation_and_delete_data_cloned(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + assets_root = os.path.normpath(settings.ASSETS_ROOT) + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertTrue(os.path.exists(file), "Asset file does not exist") + self.assertTrue( + os.path.normpath(file).startswith(os.path.normpath(assets_root)), + f"Asset file is not inside the assets root: {file}", + ) + + cloned_file = file + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertFalse(os.path.exists(cloned_file)) + self.assertFalse(os.path.exists(os.path.dirname(cloned_file))) + self.assertTrue(os.path.exists(ONE_JSON)) + + def test_creation_and_delete_data_external(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertEqual(ONE_JSON, file) + + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertTrue(os.path.exists(ONE_JSON)) + + def test_clone_and_delete_data_managed(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + self.assertNotEqual(reloaded.pk, cloned.pk) + + reloaded_file = os.path.normpath(reloaded.location[0]) + cloned_file = os.path.normpath(cloned.location[0]) + + self.assertNotEqual(reloaded_file, cloned_file) + self.assertTrue(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + reloaded.delete() + self.assertFalse(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + cloned.delete() + self.assertFalse(os.path.exists(cloned_file)) + + def test_clone_and_delete_data_unmanaged(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + + self.assertEqual(reloaded.location[0], cloned.location[0]) + self.assertTrue(os.path.exists(reloaded.location[0])) + + reloaded.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + cloned.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + def test_clone_mixed_data(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + managed_asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + managed_asset.save() + + # TODO: dunno if mixed files should be allowed at all + mixed_asset = asset_handler.create( + title="Mixed Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[THREE_JSON, managed_asset.location[0]], + clone_files=False, # let's keep both managed and unmanaged together + ) + + reloaded = Asset.objects.get(pk=mixed_asset.pk) + + try: + asset_handler.clone(reloaded) + self.fail("A mixed LocalAsset has been cloned") + except ValueError: + pass + + managed_asset.delete() + + try: + mixed_asset.delete() + self.fail("Missed mixed LocalAsset detection") + except ValueError: + pass + + +class AssetsDownloadTests(APITestCase): + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def _get_streaming_content(self, response): + with io.BytesIO(b"".join(response.streaming_content)) as buf_bytes: + return buf_bytes.read() + + def test_download_file(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + self.assertTrue(self.client.login(username="admin", password="admin"), "Login failed") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + + # put two more files in the asset dir + asset_dir = os.path.dirname(reloaded.location[0]) + sub_dir = os.path.join(asset_dir, "subdir") + os.mkdir(sub_dir) + shutil.copy(TWO_JSON, asset_dir) + shutil.copy(THREE_JSON, sub_dir) + + for path, key in ((None, "one"), ("one.json", "one"), ("two.json", "two"), ("subdir/three.json", "three")): + # args = [asset.pk, path] if path else [asset.pk] + args = {"pk": asset.pk, "path": path} if path else {"pk": asset.pk} + logger.info(f"*** Testing path '{path}' args {args}") + url = reverse("assets-link", kwargs=args) + logger.info(f"REVERSE url is {url}") + response = self.client.get(url) + content = self._get_streaming_content(response) + rjson = json.loads(content) + self.assertEqual(response.status_code, 200) + self.assertIn(key, rjson, f"Key '{key}' not found in path '{path}': {rjson} URL {url}") + logger.info(f"Test for path '{path}' OK") diff --git a/geonode/assets/tests/data/one.json b/geonode/assets/tests/data/one.json new file mode 100644 index 00000000000..27a1ec027ca --- /dev/null +++ b/geonode/assets/tests/data/one.json @@ -0,0 +1,3 @@ +{ + "one": 1 +} \ No newline at end of file diff --git a/geonode/assets/tests/data/three.json b/geonode/assets/tests/data/three.json new file mode 100644 index 00000000000..8ced1fb5978 --- /dev/null +++ b/geonode/assets/tests/data/three.json @@ -0,0 +1,3 @@ +{ + "three": 3 +} \ No newline at end of file diff --git a/geonode/assets/tests/data/two.json b/geonode/assets/tests/data/two.json new file mode 100644 index 00000000000..c8b2fcf3d89 --- /dev/null +++ b/geonode/assets/tests/data/two.json @@ -0,0 +1,3 @@ +{ + "two": 2 +} \ No newline at end of file diff --git a/geonode/assets/urls.py b/geonode/assets/urls.py new file mode 100644 index 00000000000..956854d2523 --- /dev/null +++ b/geonode/assets/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from geonode.assets import views + +router.register(r"assets", views.AssetViewSet, "assets") + +urlpatterns = [] diff --git a/geonode/assets/utils.py b/geonode/assets/utils.py new file mode 100644 index 00000000000..06abd560ea0 --- /dev/null +++ b/geonode/assets/utils.py @@ -0,0 +1,163 @@ +import logging +import os.path + +from django.http import HttpResponse + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.models import Asset +from geonode.base.models import ResourceBase, Link +from geonode.security.utils import get_visible_resources + + +logger = logging.getLogger(__name__) + + +def get_perms_response(request, asset: Asset): + user = request.user + + # quick check + is_admin = user.is_superuser if user and user.is_authenticated else False + if is_admin or user == asset.owner: + logger.debug("Asset: access allowed by user") + return None + + visibile_res = get_visible_resources(queryset=ResourceBase.objects.filter(link__asset=asset), user=request.user) + + logger.warning("TODO: implement permission check") + if visibile_res.exists(): + logger.debug("Asset: access allowed by Resource") + return None + elif user and user.is_authenticated: + return HttpResponse(status=403) + else: + return HttpResponse(status=401) + + +def get_default_asset(resource: ResourceBase, link_type=None) -> Asset or None: + """ + Get the default asset for a ResourceBase. + + In this first implementation we select the first one -- + in the future there may be further flags to identify the preferred one + """ + filters = {"link__resource": resource} + if link_type: + filters["link__link_type"] = link_type + + return Asset.objects.filter(**filters).first() + + +DEFAULT_TYPES = {"image": ["jpg", "jpeg", "gif", "png", "bmp", "svg"]} + + +def find_type(ext): + return next((datatype for datatype, extensions in DEFAULT_TYPES.items() if ext.lower() in extensions), None) + + +def create_link(resource, asset, link_type=None, extension=None, name=None, mime=None, asset_handler=None, **kwargs): + asset_handler = asset_handler or asset_handler_registry.get_handler(asset) + + if not link_type or not extension or not name: + fallback_name, fallback_ext = ( + os.path.splitext(os.path.basename(asset.location[0])) if len(asset.location) else (None, None) + ) + if fallback_ext: + fallback_ext = fallback_ext.lstrip(".") + link_type = link_type or find_type(fallback_ext) if fallback_ext else None + + link = Link( + resource=resource, + asset=asset, + url=asset_handler.create_link_url(asset), + extension=extension or fallback_ext or "Unknown", + link_type=link_type or "data", + name=name or fallback_name or asset.title, + mime=mime or "", + ) + link.save() + return link + + +def create_asset_and_link( + resource, + owner, + files: list, + handler=None, + title=None, + description=None, + link_type=None, + extension=None, + asset_type=None, + mime=None, + clone_files: bool = True, +) -> tuple[Asset, Link]: + + asset_handler = handler or asset_handler_registry.get_default_handler() + asset = link = None + try: + default_title, default_ext = os.path.splitext(next(f for f in files)) if len(files) else (None, None) + if default_ext: + default_ext = default_ext.lstrip(".") + link_type = link_type or find_type(default_ext) if default_ext else None + + asset = asset_handler.create( + title=title or default_title or "Unknown", + description=description or asset_type or "Unknown", + type=asset_type or "Unknown", + owner=owner, + files=files, + clone_files=clone_files, + ) + + link = create_link( + resource, + asset, + asset_handler=asset_handler, + link_type=link_type, + extension=extension, + name=title, + mime=mime, + ) + + return asset, link + except Exception as e: + logger.error(f"Error creating Asset for resource {resource}: {e}", exc_info=e) + rollback_asset_and_link(asset, link) + raise Exception(f"Error creating asset: {e}") + + +def create_asset_and_link_dict(resource, values: dict, clone_files=True): + return create_asset_and_link( + resource, + values["owner"], + values["files"], + title=values.pop("data_title", None), + description=values.pop("description", None), + link_type=values.pop("link_type", None), + extension=values.pop("extension", None), + asset_type=values.pop("data_type", None), + clone_files=clone_files, + ) + + +def copy_assets_and_links(resource, target=None) -> list: + assets_and_links = [] + links_with_assets = Link.objects.filter(resource=resource, asset__isnull=False).prefetch_related("asset") + + for link in links_with_assets: + link.asset = asset_handler_registry.get_handler(link.asset).clone(link.asset) + link.pk = None + link.resource = target + link.save() + assets_and_links.append((link.asset, link)) + return assets_and_links + + +def rollback_asset_and_link(asset, link): + try: + if link: + link.delete() + if asset: + asset.delete() # TODO: make sure we are only deleting from DB and not also the stored data + except Exception as e: + logger.error(f"Could not rollback asset[{asset}] and link[{link}]", exc_info=e) diff --git a/geonode/assets/views.py b/geonode/assets/views.py new file mode 100644 index 00000000000..cc47b861343 --- /dev/null +++ b/geonode/assets/views.py @@ -0,0 +1,113 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.shortcuts import get_object_or_404 +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.serializers import AssetSerializer +from geonode.assets.utils import get_perms_response +from geonode.assets.models import Asset + +from geonode.base.api.filters import ( + DynamicSearchFilter, +) +from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.permissions import UserHasPerms + +logger = logging.getLogger(__name__) + + +class AssetViewSet(DynamicModelViewSet): + """ + API endpoint that allows Assets to be viewed or edited. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + # TODO: add filtering by owner / admin + ] + queryset = Asset.objects.all().order_by("-created") + serializer_class = AssetSerializer # TODO: appropriate Serializer should be switched for each Asset instance + pagination_class = GeoNodeApiPagination + + def list(self, request, *args, **kwargs): + """ + Only for lists, allows access to Assets only to owned ones, or to all of them if the user is an admin + """ + queryset = self.filter_queryset(self.get_queryset()) + + user = request.user + is_admin = user.is_superuser if user and user.is_authenticated else False + + if is_admin: + pass + elif user and user.is_authenticated: + queryset = queryset.filter(owner=user) + else: + queryset = queryset.none() + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + def _get_file(self, request, pk, attachment: bool = False, path=None): + asset = get_object_or_404(Asset, pk=pk) + if bad_response := get_perms_response(request, asset): + return bad_response + asset_handler = asset_handler_registry.get_handler(asset) + # TODO: register_event(request, EventType.EVENT_DOWNLOAD, asset) + return asset_handler.get_download_handler(asset).create_response(asset, path=path, attachment=attachment) + + @action( + detail=False, + url_path="(?P\d+)/download(/(?P.*))?", # noqa + # url_name="asset-download", + methods=["get"], + ) + def download(self, request, pk=None, path=None, *args, **kwargs): + return self._get_file(request, pk, attachment=True, path=path) + + @action( + detail=False, + url_path="(?P\d+)/link(?:/(?P.*))?", # noqa + # url_name="asset-link", + methods=["get"], + ) + def link(self, request, pk=None, path=None, *args, **kwargs): + logger.warning(f"REQUESTED ASSET LINK FOR PK:{pk} PATH:{path}") + return self._get_file(request, pk, attachment=False, path=path) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 5aa9e42d7fc..f042fe45032 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -28,6 +28,7 @@ from django.forms.models import model_to_dict from django.contrib.auth import get_user_model from django.db.models.query import QuerySet +from geonode.assets.utils import get_default_asset from geonode.people import Roles from django.http import QueryDict from deprecated import deprecated @@ -62,7 +63,8 @@ from geonode.geoapps.models import GeoApp from geonode.groups.models import GroupCategory, GroupProfile from geonode.base.api.fields import ComplexDynamicRelationField -from geonode.layers.utils import get_dataset_download_handlers, get_default_dataset_download_handler +from geonode.layers.utils import get_download_handlers, get_default_dataset_download_handler +from geonode.assets.handlers import asset_handler_registry from geonode.utils import build_absolute_uri from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes from geonode.resource.models import ExecutionRequest @@ -298,15 +300,24 @@ def get_attribute(self, instance): except Exception as e: logger.exception(e) raise e + + asset = get_default_asset(_instance) + if asset is not None: + asset_url = asset_handler_registry.get_handler(asset).create_download_url(asset) + if _instance.resource_type in ["map"] + get_geoapp_subtypes(): return [] elif _instance.resource_type in ["document"]: - return [ + payload = [ { "url": _instance.download_url, "ajax_safe": _instance.download_is_ajax_safe, - } + }, ] + if asset: + payload.append({"url": asset_url, "ajax_safe": False, "default": False}) + return payload + elif _instance.resource_type in ["dataset"]: download_urls = [] # lets get only the default one first to set it @@ -315,11 +326,14 @@ def get_attribute(self, instance): if obj.download_url: download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": True}) # then let's prepare the payload with everything - handler_list = get_dataset_download_handlers() - for handler in handler_list: + for handler in get_download_handlers(): obj = handler(self.context.get("request"), _instance.alternate) if obj.download_url: download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False}) + + if asset: + download_urls.append({"url": asset_url, "ajax_safe": True, "default": False if download_urls else True}) + return download_urls else: return [] @@ -524,7 +538,7 @@ def to_representation(self, instance): ret = [] link_fields = ["extension", "link_type", "name", "mime", "url"] links = Link.objects.filter( - resource_id=instance, link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS", "image", "metadata"] + resource_id=instance, # link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS", "image", "metadata"] ) for lnk in links: formatted_link = model_to_dict(lnk, fields=link_fields) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index be0ae7c1ff0..0955916cb09 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -22,11 +22,11 @@ import sys import json import logging +from builtins import Exception from typing import Iterable from django.test import RequestFactory, override_settings import gisdata - from PIL import Image from io import BytesIO from time import sleep @@ -43,10 +43,14 @@ from rest_framework.test import APITestCase from rest_framework.renderers import JSONRenderer from rest_framework.parsers import JSONParser - +from geonode.resource.manager import resource_manager from guardian.shortcuts import get_anonymous_user + +from geonode.assets.utils import create_asset_and_link from geonode.maps.models import Map, MapLayer from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.assets.utils import get_default_asset +from geonode.assets.handlers import asset_handler_registry from geonode.base import enumerations from geonode.base.api.serializers import ResourceBaseSerializer @@ -2221,16 +2225,22 @@ def test_manager_can_edit_map(self): ) def test_resource_service_copy(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) - resource = Dataset.objects.create( - owner=get_user_model().objects.get(username="admin"), - name="test_copy", - store="geonode_data", - subtype="vector", - alternate="geonode:test_copy", - uuid=str(uuid4()), - files=list(files_as_dict.values()), + resource = resource_manager.create( + str(uuid4()), + Dataset, + defaults={ + "owner": get_user_model().objects.get(username="admin"), + "name": "test_copy", + "store": "geonode_data", + "subtype": "vector", + "alternate": "geonode:test_copy", + }, + ) + + asset, link = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) ) bobby = get_user_model().objects.get(username="bobby") copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) @@ -2262,22 +2272,29 @@ def test_resource_service_copy(self): cloned_resource = Dataset.objects.last() self.assertEqual(cloned_resource.owner.username, "admin") # clone dataset with invalid file - resource.files = ["/path/invalid_file.wrong"] - resource.save() + # resource.files = ["/path/invalid_file.wrong"] + # resource.save() + asset.location = ["/path/invalid_file.wrong"] + asset.save() response = self.client.put(copy_url) + self.assertEqual(response.status_code, 400) self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clone dataset with no files - resource.files = [] - resource.save() + link.delete() + asset.delete() response = self.client.put(copy_url) + self.assertEqual(response.status_code, 400) self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clean - resource.delete() + try: + resource.delete() + except Exception as e: + logger.warning(f"Can't delete test resource {resource}", exc_info=e) def test_resource_service_copy_with_perms_dataset(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Dataset.objects.create( owner=get_user_model().objects.get(username="admin"), @@ -2287,7 +2304,9 @@ def test_resource_service_copy_with_perms_dataset(self): alternate="geonode:test_copy", resource_type="dataset", uuid=str(uuid4()), - files=list(files_as_dict.values()), + ) + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) ) self._assertCloningWithPerms(resource) @@ -2295,21 +2314,25 @@ def test_resource_service_copy_with_perms_dataset(self): @override_settings(ASYNC_SIGNALS=False) def test_resource_service_copy_with_perms_dataset_set_default_perms(self): with self.settings(ASYNC_SIGNALS=False): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) - resource = Dataset.objects.create( - owner=get_user_model().objects.get(username="admin"), - name="test_copy_with_perms", - store="geonode_data", - subtype="vector", - alternate="geonode:test_copy_with_perms", - resource_type="dataset", - uuid=str(uuid4()), - files=list(files_as_dict.values()), + resource = resource_manager.create( + None, + resource_type=Dataset, + defaults={ + "owner": get_user_model().objects.first(), + "title": "test_copy_with_perms", + "name": "test_copy_with_perms", + "is_approved": True, + "store": "geonode_data", + "subtype": "vector", + "resource_type": "dataset", + "files": files_as_dict.values(), + }, ) _perms = { "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, - "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebae"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebase"]}, } resource.set_permissions(_perms) # checking that bobby is in the original dataset perms list @@ -2328,11 +2351,11 @@ def test_resource_service_copy_with_perms_dataset_set_default_perms(self): self.assertEqual("finished", self.client.get(response.json().get("status_url")).json().get("status")) _resource = Dataset.objects.filter(title__icontains="test_copy_with_perms").last() self.assertIsNotNone(_resource) - self.assertFalse("bobby" in "bobby" in [x.username for x in _resource.get_all_level_info().get("users", [])]) - self.assertTrue("admin" in "admin" in [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertNotIn("bobby", [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertIn("admin", [x.username for x in _resource.get_all_level_info().get("users", [])]) def test_resource_service_copy_with_perms_doc(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( owner=get_user_model().objects.get(username="admin"), @@ -2340,23 +2363,25 @@ def test_resource_service_copy_with_perms_doc(self): alternate="geonode:test_copy", resource_type="document", uuid=str(uuid4()), - files=list(files_as_dict.values()), ) - + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) self._assertCloningWithPerms(resource) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_resource_service_copy_with_perms_map(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( owner=get_user_model().objects.get(username="admin"), alternate="geonode:test_copy", resource_type="map", uuid=str(uuid4()), - files=list(files_as_dict.values()), ) - + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) self._assertCloningWithPerms(resource) def _assertCloningWithPerms(self, resource): @@ -2454,7 +2479,12 @@ def test_base_resources_return_download_links_for_documents(self): Ensure we can access the Resource Base list. """ doc = Document.objects.first() - expected_payload = [{"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}] + asset = get_default_asset(doc) + handler = asset_handler_registry.get_handler(asset) + expected_payload = [ + {"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}, + {"ajax_safe": False, "default": False, "url": handler.create_download_url(asset)}, + ] # From resource base API json = self._get_for_object(doc, "base-resources-detail") download_url = json.get("resource").get("download_urls") diff --git a/geonode/base/base_urls.py b/geonode/base/base_urls.py new file mode 100644 index 00000000000..8d97519805f --- /dev/null +++ b/geonode/base/base_urls.py @@ -0,0 +1,46 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import re_path + +from . import views + +js_info_dict = { + "packages": ("geonode.base",), +} + +urlpatterns = [ + # 'geonode.resourcebases.views', + re_path(r"^(?P\d+)/metadata$", views.resourcebase_metadata, name="resourcebase_metadata"), + re_path( + r"^(?P[^/]*)/metadata_detail$", + views.resourcebase_metadata_detail, + name="resourcebase_metadata_detail", + ), + re_path( + r"^(?P\d+)/metadata_advanced$", + views.resourcebase_metadata_advanced, + name="resourcebase_metadata_advanced", + ), + re_path( + r"^(?P[^/]+)/embed$", + views.resourcebase_embed, + {"template": "base/base_embed.html"}, + name="resourcebase_embed", + ), +] diff --git a/geonode/base/enumerations.py b/geonode/base/enumerations.py index 4d956e8331f..66430b48bd5 100644 --- a/geonode/base/enumerations.py +++ b/geonode/base/enumerations.py @@ -19,7 +19,7 @@ from django.utils.translation import gettext_lazy as _ -LINK_TYPES = ["original", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] +LINK_TYPES = ["original", "uploaded", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] HIERARCHY_LEVELS = ( ("series", _("series")), diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 14552bed7d5..6e01854d2ce 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -623,6 +623,8 @@ def clean_extra_metadata(self): return json.dumps(validate_extra_metadata(cleaned_data, self.instance), indent=4) class Meta: + model = ResourceBase + exclude = ( "contacts", "name", diff --git a/geonode/base/migrations/0091_create_link_asset_alter_link_type.py b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py new file mode 100644 index 00000000000..e3310ca764d --- /dev/null +++ b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0091_alter_hierarchicalkeyword_slug"), + ("assets", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="asset", + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="assets.asset"), + ), + migrations.AlterField( + model_name="link", + name="link_type", + field=models.CharField( + choices=[ + ("original", "original"), + ("uploaded", "uploaded"), + ("data", "data"), + ("image", "image"), + ("metadata", "metadata"), + ("html", "html"), + ("OGC:WMS", "OGC:WMS"), + ("OGC:WFS", "OGC:WFS"), + ("OGC:WCS", "OGC:WCS"), + ], + max_length=255, + ), + ), + ] diff --git a/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py new file mode 100644 index 00000000000..76b564cd918 --- /dev/null +++ b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py @@ -0,0 +1,95 @@ +# Generated by Django 4.2.9 on 2024-03-12 11:55 +import logging +import os + +from django.db import migrations +from django.db.models import Q +from django.conf import settings +from django.contrib.auth import get_user_model +from django.urls import reverse + +from geonode.base.models import Link +from geonode.assets.models import LocalAsset +from geonode.utils import build_absolute_uri + +logger = logging.getLogger(__name__) + + +def migrate_files(apps, schema_editor): + + def get_ext(filename): + try: + return os.path.splitext(filename)[1][1:] + except Exception as e: + logger.warning(f"Could not find extension for Resource '{res_hm.title}, file '{filename}': {e}") + return None + + ResourceBase_hm = apps.get_model('base', 'ResourceBase') + Dataset_hm = apps.get_model('layers', 'Dataset') + Document_hm = apps.get_model('documents', 'Document') + + if hasattr(ResourceBase_hm, "files"): + # looping on available resources with files to generate the LocalAssets + for res_hm in ResourceBase_hm.objects.exclude(Q(files__isnull=True) | Q(files__exact=[])).iterator(): + # resolving the real owner instance, since resource.owner is an historical model and cant be used directly + owner = get_user_model().objects.get(pk=res_hm.owner.id) + # logger.warning(f"Creating ASSET for {resource.id} -- owner:{type(resource.owner)} --> {resource.owner}") + + files = res_hm.files + # creating the local asset object + asset = LocalAsset( + title="Files", + description="Original uploaded files", + owner=owner, + location=files + ) + asset.save() + + ### creating the association between asset and Link + + # no existing "uploaded" links exist, so create them right away + # otherwise we create the link with the assigned asset + if dataset_hm := Dataset_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + elif doc_hm := Document_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + else: + raise TypeError(f'ResourceBase {res_hm.id}::"{res_hm.title} has unhandled type"') + + if len(files) == 1: + ext = get_ext(files[0]) + else: + ext = None + for file in files: + for filetype in settings.SUPPORTED_DATASET_FILE_TYPES: + file_ext = get_ext(file) + if file_ext in filetype["ext"]: + ext = filetype["id"] + break + if ext: + break + + Link.objects.create( + resource_id=res_hm.id, + asset=asset, + link_type="uploaded", + name="Original upload", + extension=ext or "unknown", + url=url + ) + + +class Migration(migrations.Migration): + + dependencies = [ + + ("base", "0091_create_link_asset_alter_link_type"), + ] + + operations = [ + migrations.RunPython(migrate_files, migrations.RunPython.noop), + migrations.RemoveField( + model_name="resourcebase", + name="files", + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index 0c83f340898..590246505ad 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -49,6 +49,7 @@ from django.utils.translation import gettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.utils.html import strip_tags +from django.urls import reverse from mptt.models import MPTTModel, TreeForeignKey from PIL import Image, ImageOps @@ -88,7 +89,6 @@ from urllib.parse import urlsplit, urljoin from geonode.storage.manager import storage_manager - logger = logging.getLogger(__name__) @@ -607,26 +607,13 @@ def upload_files(resource_id, files, force=False): @staticmethod def cleanup_uploaded_files(resource_id): """Remove uploaded files, if any""" + from geonode.assets.utils import get_default_asset + if ResourceBase.objects.filter(id=resource_id).exists(): _resource = ResourceBase.objects.filter(id=resource_id).get() - _uploaded_folder = None - if _resource.files: - for _file in _resource.files: - try: - if storage_manager.exists(_file): - if not _uploaded_folder: - _uploaded_folder = os.path.split(storage_manager.path(_file))[0] - storage_manager.delete(_file) - except Exception as e: - logger.warning(e) - try: - if _uploaded_folder and storage_manager.exists(_uploaded_folder): - storage_manager.delete(_uploaded_folder) - except Exception as e: - logger.warning(e) - - # Do we want to delete the files also from the resource? - ResourceBase.objects.filter(id=resource_id).update(files={}) + asset = get_default_asset(_resource) # TODO: make sure to select the proper "uploaded" asset + if asset: + asset.delete() # Remove generated thumbnails, if any filename = f"{_resource.get_real_instance().resource_type}-{_resource.get_real_instance().uuid}" @@ -904,8 +891,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): _("Metadata"), default=False, help_text=_("If true, will be excluded from search") ) - files = JSONField(null=True, default=list, blank=True) - blob = JSONField(null=True, default=dict, blank=True) subtype = models.CharField(max_length=128, null=True, blank=True) @@ -1086,6 +1071,11 @@ def delete(self, notify=True, *args, **kwargs): resource_manager.remove_permissions(self.uuid, instance=self.get_real_instance()) + # delete assets. TODO: when standalone Assets will be allowed, only dependable Assets shall be removed + links_with_assets = Link.objects.filter(resource=self, asset__isnull=False).prefetch_related("asset") + for link in links_with_assets: + link.asset.delete() + if hasattr(self, "class_name") and notify: notice_type_label = f"{self.class_name.lower()}_deleted" recipients = get_notification_recipients(notice_type_label, resource=self) @@ -1278,10 +1268,12 @@ def instance_is_processed(self): @property def is_copyable(self): - from geonode.geoserver.helpers import select_relevant_files - if self.resource_type == "dataset": - allowed_file = select_relevant_files(get_allowed_extensions(), self.files) + from geonode.assets.utils import get_default_asset + from geonode.geoserver.helpers import select_relevant_files + + asset = get_default_asset(self) # TODO: maybe we need to filter by original files + allowed_file = select_relevant_files(get_allowed_extensions(), asset.location) if asset else [] return len(allowed_file) != 0 return True @@ -1339,8 +1331,14 @@ def keyword_csv(self): return "" def get_absolute_url(self): + from geonode.client.hooks import hookset + try: - return self.get_real_instance().get_absolute_url() if self != self.get_real_instance() else None + return ( + self.get_real_instance().get_absolute_url() + if self != self.get_real_instance() + else hookset.get_absolute_url(self) + ) except Exception as e: logger.exception(e) return None @@ -1461,7 +1459,11 @@ def download_links(self): @property def embed_url(self): - return self.get_real_instance().embed_url if self != self.get_real_instance() else None + return ( + self.get_real_instance().embed_url + if self != self.get_real_instance() + else reverse("resourcebase_embed", kwargs={"resourcebaseid": self.pk}) + ) def get_tiles_url(self): """Return URL for Z/Y/X mapping clients or None if it does not exist.""" @@ -2024,6 +2026,7 @@ class Link(models.Model): name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"')) mime = models.CharField(max_length=255, help_text=_('For example "text/xml"')) url = models.TextField(max_length=1000) + asset = models.ForeignKey("assets.Asset", null=True, on_delete=models.CASCADE) objects = LinkManager() diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index faa8c3b91f6..d8cc78afd18 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -26,7 +26,6 @@ from taggit.models import TaggedItem from datetime import datetime, timedelta -from django.conf import settings from django.db import transaction from django.utils import timezone from django.db.utils import IntegrityError @@ -36,6 +35,7 @@ from django.contrib.auth.models import Permission, Group from django.core.files.uploadedfile import SimpleUploadedFile +from geonode.assets.utils import create_asset_and_link from geonode.maps.models import Map from geonode.base import enumerations from geonode.layers.models import Dataset @@ -54,7 +54,7 @@ b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" ) f = SimpleUploadedFile("test_img_file.gif", imgfile.read(), "image/gif") -dfile = [f"{settings.MEDIA_ROOT}/img.gif"] +dfile = [f"{os.path.dirname(__file__)}/tests/data/img.gif"] def all_public(): @@ -268,11 +268,12 @@ def create_models(type=None, integration=False): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, + # files=dfile, extension="gif", metadata_only=title == "doc metadata true", ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=user) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) @@ -472,11 +473,11 @@ def create_single_doc(name, owner=None, **kwargs): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, resource_type="document", **kwargs, ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) @@ -504,10 +505,10 @@ def create_single_geoapp(name, resource_type="geostory", owner=None, **kwargs): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, **kwargs, ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) diff --git a/geonode/base/templates/base/base_embed.html b/geonode/base/templates/base/base_embed.html new file mode 100644 index 00000000000..fc2f4773841 --- /dev/null +++ b/geonode/base/templates/base/base_embed.html @@ -0,0 +1,14 @@ +{% load i18n %} +{% load base_tags %} +{% load client_lib_tags %} + +{% block head %} + + + + {% get_resourcebase_embed %} +{% endblock %} \ No newline at end of file diff --git a/geonode/base/templates/base/base_metadata.html b/geonode/base/templates/base/base_metadata.html new file mode 100644 index 00000000000..202ff746115 --- /dev/null +++ b/geonode/base/templates/base/base_metadata.html @@ -0,0 +1,87 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load floppyforms %} + +{% block title %}{{ resource.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} + + + +
+ {% if resource.metadata_uploaded %} +
{% blocktrans %}Note: this resource's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
+ {% endif %} + + {% if resourcebase_form.errors or category_form.errors or tkeywords_form.errors %} +
{% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
    + {% for field in resourcebase_form %} + {% if field.errors %} +
  • {{ field.label }}
  • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
  • {{ category_form.errors.as_ul }}
  • + {% endif %} + {% if tkeywords_form.errors %} +
  • {{ tkeywords_form.errors.as_ul }}
  • + {% endif %} +
+
+ {% endif %} + + {% csrf_token %} +
+ {% form resourcebase_form using panel_template %} + {# resourcebase_form|as_bootstrap #} +
+ +
+
+ + + +
+ + + >" %}"/> +
+
+
+
+ + + +{{ block.super }} +{% endblock body_outer %} diff --git a/geonode/base/templates/base/base_metadata_advanced.html b/geonode/base/templates/base/base_metadata_advanced.html new file mode 100644 index 00000000000..8eba9be1d18 --- /dev/null +++ b/geonode/base/templates/base/base_metadata_advanced.html @@ -0,0 +1,134 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load base_tags %} +{% load bootstrap_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} +{% block title %}{{ resourcebase.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock %} + +{% block body_outer %} + +{{ block.super }} + + + + + + + + + + + +
+
+

+ {% blocktrans with resourcebase.title as map_title %} + Editing details for {{ map_title }} + {% endblocktrans %} +

+ +
+ {% if resourcebase.metadata_uploaded %} +
{% blocktrans %}Note: this resourcebase's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
+ {% endif %} + + {% if resourcebase_form.errors or category_form.errors %} +
{% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
    + {% for field in resourcebase_form %} + {% if field.errors %} +
  • {{ field.label }}
  • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
  • {{ category_form.errors.as_ul }}
  • + {% endif %} +
+
+ {% endif %} +
+ +
+ {% csrf_token %} + +
+ {% block resourcebase_fields %} + {% for field in resourcebase_form %} + {% if field.name != 'use_featureinfo_custom_template' and field.name != 'featureinfo_custom_template' and field.name not in ADVANCED_EDIT_EXCLUDE_FIELD %} + {% if field.name == 'featured' and not user.is_superuser %} + {% else %} +
+
+ + {{ field }} +
+
+ {% endif %} + {% endif %} + {% endfor %} + {% endblock resourcebase_fields %} + + + {% block thesauri %} + {% if THESAURI_FILTERS %} + {% for field in tkeywords_form %} +
+

+ + {{ field }} +

+
+ {% endfor %} + {% endif %} + {% endblock thesauri %} +
+
+
+ +
+ {% autoescape off %} + {% for choice in category_form.category_choice_field.field.choices %} +
+ +
+ {% endfor %} + {% endautoescape %} +
+
+ +
+ + + +
+ +
+
+
+
+
+
+{% endblock %} diff --git a/geonode/base/templates/base/base_metadata_detail.html b/geonode/base/templates/base/base_metadata_detail.html new file mode 100644 index 00000000000..a1c62b43517 --- /dev/null +++ b/geonode/base/templates/base/base_metadata_detail.html @@ -0,0 +1,6 @@ +{% extends "metadata_detail.html" %} +{% load i18n %} +{% block metaget_absolute_url %} +
{% trans "Metadata Page" %}
+
{% url "resourcebase_metadata_detail" resource.id %}
+{% endblock metaget_absolute_url %} \ No newline at end of file diff --git a/geonode/base/templates/base/base_panels.html b/geonode/base/templates/base/base_panels.html new file mode 100644 index 00000000000..e82cf6035eb --- /dev/null +++ b/geonode/base/templates/base/base_panels.html @@ -0,0 +1,607 @@ +{% load i18n %} +{% load static %} +{% load floppyforms %} +{% load contact_roles %} + + + + + + + + + + + + + + + + + + +{% block body_outer %} + + +
+
+
+ +
+ {% trans "Mandatory" %} +
+
+ {% trans "Mandatory" %} +
+
+ {% trans "Optional" %} +
+
+ +
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+
+ + +
+
+ {% block resourcebase_title %} + + + {{ resourcebase_form.title }} + {% endblock %} +
+ + + {{ resourcebase_form.abstract }} +
+
+
+
+ + + {{ resourcebase_form.date_type }} +
+
+ + + {{ resourcebase_form.date }} +
+ {% block resourcebase_category %} +
+ + +
+ {% endblock resourcebase_category %} +
+ + +
+
+ + {{ resourcebase_form.keywords }} +
+ {% if THESAURI_FILTERS %} +
+ {{tkeywords_form.as_p}} +
+ {% endif %} +
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+ {% block resourcebase_attributes %} +
+
+ + + {{ resourcebase_form.language }} +
+
+ + + {{ resourcebase_form.license }} +
+
+ + {{ resourcebase_form.attribution }} +
+
+ {% endblock resourcebase_attributes %} +
+
+ + {{ resourcebase_form.regions }} +
+
+ + + {{ resourcebase_form.data_quality_statement }} +
+
+
+
+ + + {{ resourcebase_form.restriction_code_type }} +
+
+ + + {{ resourcebase_form.constraints_other }} +
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+

{% trans "Other, Optional, Metadata" %}

+
+ + + {{ resourcebase_form.edition }} +
+
+ + {{ resourcebase_form.doi }} +
+
+ + + {{ resourcebase_form.purpose }} +
+
+ + + {{ resourcebase_form.supplemental_information }} +
+
+
+ {% block resourcebase_temporal_extent_start %} +
+
+ + + {{ resourcebase_form.temporal_extent_start }} +
+
+ {% endblock resourcebase_temporal_extent_start %} + {% block resourcebase_temporal_extent_end %} +
+
+ + + {{ resourcebase_form.temporal_extent_end }} +
+
+ {% endblock resourcebase_temporal_extent_end %} + {% block maintenance_block %} +
+
+ + + {{ resourcebase_form.maintenance_frequency }} +
+
+ + + {{ resourcebase_form.spatial_representation_type }} +
+ {% block resourcebase_extra_metadata %} +
+ + {{ resourcebase_form.extra_metadata }} +
+ {% endblock resourcebase_extra_metadata %} + {% block resourcebase_linked_resources %} +
+ + {{ resourcebase_form.linked_resources }} +
+ {% endblock resourcebase_linked_resources %} + +
+ {% endblock maintenance_block %} +
+
+ {% block resourcebase_poc %} +
+
{% trans "Responsible Parties" %}
+
+ + {{ resourcebase_form.poc }} +
+
+ {% endblock %} +
+
{% trans "Responsible and Permissions" %}
+ {% block resourcebase_owner %} +
+
+ + {{ resourcebase_form.owner }} +
+ {% endblock resourcebase_owner %} +
+
+ {% trans "toggle more Contact Roles" %} + {% block resourcebase_more_contact_roles %} +
+
{% trans "more metadata contact roles" %}
+ {% for contact_role in UI_ROLES_IN_TOGGLE_VIEW %} + {% getattribute resourcebase_form contact_role as cr %} +
+
+ + {{ cr}} +
+
+ {% endfor %} +
+
+ {% endblock resourcebase_more_contact_roles %} +
+ +
+
+
+ {% block extra_metadata_content %} + {% endblock %} +
+
+ +
+
+
+
+
+
{% trans "Publishing" %}
+
+
+ + {{ resourcebase_form.metadata_uploaded_preserve }} +
+
+ + {{ resourcebase_form.is_approved }} +
+
+ + {{ resourcebase_form.is_published }} +
+ {% if user.is_superuser %} +
+ + {{ resourcebase_form.featured }} +
+ {% endif %} +
+ + {{ resourcebase_form.advertised }} +
+
+
+
+
+
+
+
+
+
+
+
+
{% trans "Other Settings" %}
+
+
+ + {{ resourcebase_form.id }} +
+
+
+
+
+
+
+
+
+{% endblock %} diff --git a/geonode/base/tests/data/img.gif b/geonode/base/tests/data/img.gif new file mode 100644 index 00000000000..56959b6411a Binary files /dev/null and b/geonode/base/tests/data/img.gif differ diff --git a/geonode/base/views.py b/geonode/base/views.py index e51c342fedf..64adc5bf202 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -18,6 +18,9 @@ ######################################################################### import json import logging +import ast +import warnings +import traceback from dal import views, autocomplete from user_messages.models import Message @@ -28,6 +31,8 @@ from django.shortcuts import render from django.http import HttpResponse from django.views.generic import FormView +from django.core.exceptions import ObjectDoesNotExist +from django.views.decorators.clickjacking import xframe_options_sameorigin from django.http import HttpResponseRedirect from django.contrib.auth import get_user_model from django.contrib import messages @@ -55,6 +60,21 @@ from geonode.base.forms import BatchEditForm, OwnerRightsRequestForm from geonode.base.models import Region, ResourceBase, HierarchicalKeyword, ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.base.enumerations import SOURCE_TYPE_LOCAL + +from geonode.client.hooks import hookset +from geonode.people.forms import ProfileForm +from geonode.monitoring.models import EventType +from geonode.base.auth import get_or_create_token +from geonode.security.views import _perms_info_json +from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager +from geonode.decorators import check_keyword_write_perms + +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory + +from .forms import ResourceBaseForm + logger = logging.getLogger(__name__) @@ -479,3 +499,347 @@ def resource_clone(request): status_code = 400 return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) + + +logger = logging.getLogger("geonode.base.metadata") + +_PERMISSION_MSG_GENERIC = _("You do not have permissions for this resource.") +_PERMISSION_MSG_METADATA = _("You are not allowed to modify this resource's metadata.") +_PERMISSION_MSG_VIEW = _("You are not allowed to view this resource.") + + +def _resolve_resourcebase(request, id, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ + Resolve the resourcebase by the provided typename and check the optional permission. + """ + + return resolve_object(request, ResourceBase, {"pk": id}, permission=permission, permission_msg=msg, **kwargs) + + +@xframe_options_sameorigin +def resourcebase_embed(request, resourcebaseid, template="base/base_edit.html"): + """ + The view that returns the app composer opened to + the app with the given app ID. + """ + try: + resourcebase_obj = _resolve_resourcebase( + request, resourcebaseid, "base.view_resourcebase", _PERMISSION_MSG_VIEW + ) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + # Call this first in order to be sure "perms_list" is correct + permissions_json = _perms_info_json(resourcebase_obj) + + perms_list = list( + resourcebase_obj.get_self_resource() + .get_user_perms(request.user) + .union(resourcebase_obj.get_user_perms(request.user)) + ) + + group = None + if resourcebase_obj.group: + try: + group = GroupProfile.objects.get(slug=resourcebase_obj.group.name) + except GroupProfile.DoesNotExist: + group = None + + r = resourcebase_obj + if request.method in ("POST", "PATCH", "PUT"): + r = resource_manager.update(resourcebase_obj.uuid, instance=resourcebase_obj, notify=True) + + resource_manager.set_permissions( + resourcebase_obj.uuid, instance=resourcebase_obj, permissions=ast.literal_eval(permissions_json) + ) + + resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False) + + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and not access_token.is_expired(): + access_token = access_token.token + else: + access_token = None + + _config = json.dumps(r.blob) + _ctx = { + "appId": resourcebaseid, + "appType": resourcebase_obj.resource_type, + "config": _config, + "user": request.user, + "access_token": access_token, + "resource": resourcebase_obj, + "group": group, + "perms_list": perms_list, + "permissions_json": permissions_json, + "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"), + } + + return render(request, template, context=_ctx) + + +def resourcebase_metadata_detail( + request, resourcebaseid, template="base/base_metadata_detail.html", custom_metadata=None +): + try: + resourcebase_obj = _resolve_resourcebase(request, resourcebaseid, "view_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + group = None + if resourcebase_obj.group: + try: + group = GroupProfile.objects.get(slug=resourcebase_obj.group.name) + except ObjectDoesNotExist: + group = None + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj) + + return render( + request, + template, + context={ + "resource": resourcebase_obj, + "group": group, + "SITEURL": site_url, + "custom_metadata": custom_metadata, + }, + ) + + +@login_required +@check_keyword_write_perms +def resourcebase_metadata( + request, + resourcebaseid, + template="base/base_metadata.html", + ajax=True, + panel_template="base/base_panels.html", + custom_metadata=None, +): + resourcebase_obj = None + try: + resourcebase_obj = _resolve_resourcebase( + request, resourcebaseid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA + ) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + # Add metadata_author or poc if missing + resourcebase_obj.add_missing_metadata_author_or_poc() + resource_type = resourcebase_obj.resource_type + topic_category = resourcebase_obj.category + subtype = resourcebase_obj.subtype + current_keywords = [keyword.name for keyword in resourcebase_obj.keywords.all()] + + topic_thesaurus = resourcebase_obj.tkeywords.all() + + if request.method == "POST": + resourcebase_form = ResourceBaseForm( + request.POST, instance=resourcebase_obj, prefix="resource", user=request.user + ) + category_form = CategoryForm( + request.POST, + prefix="category_choice_field", + initial=( + int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None + ), + ) + + if hasattr(settings, "THESAURUS"): + tkeywords_form = TKeywordForm(request.POST) + else: + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") + + else: + resourcebase_form = ResourceBaseForm(instance=resourcebase_obj, prefix="resource", user=request.user) + resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) + category_form = CategoryForm( + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) + + # Create THESAURUS widgets + lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en" + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + dataset_tkeywords = resourcebase_obj.tkeywords.all() + tkeywords_list = "" + if dataset_tkeywords and len(dataset_tkeywords) > 0: + tkeywords_ids = dataset_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + el = settings.THESAURUS + thesaurus_name = el["name"] + try: + t = Thesaurus.objects.get(identifier=thesaurus_name) + for tk in t.thesaurus.filter(pk__in=tkeywords_ids): + tkl = tk.keyword.filter(lang=lang) + if len(tkl) > 0: + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids + except Exception: + tb = traceback.format_exc() + logger.error(tb) + tkeywords_form = TKeywordForm(instance=resourcebase_obj) + else: + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") + # set initial values for thesaurus form + for tid in tkeywords_form.fields: + values = [] + values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id] + tkeywords_form.fields[tid].initial = values + + if ( + request.method == "POST" + and resourcebase_form.is_valid() + and category_form.is_valid() + and tkeywords_form.is_valid() + ): + new_keywords = current_keywords if request.keyword_readonly else resourcebase_form.cleaned_data.pop("keywords") + new_regions = resourcebase_form.cleaned_data.pop("regions") + + new_category = None + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) + resourcebase_form.cleaned_data.pop("ptype") + + resourcebase_obj = resourcebase_form.instance + # update contact roles + resourcebase_obj.set_contact_roles_from_metadata_edit(resourcebase_form) + + vals = dict(category=new_category, subtype=subtype) + + resourcebase_form.cleaned_data.pop("metadata") + extra_metadata = resourcebase_form.cleaned_data.pop("extra_metadata") + + resourcebase_form.save_linked_resources() + resourcebase_form.cleaned_data.pop("linked_resources") + + vals.update({"resource_type": resource_type, "sourcetype": SOURCE_TYPE_LOCAL}) + + register_event(request, EventType.EVENT_CHANGE_METADATA, resourcebase_obj) + if not ajax: + return HttpResponseRedirect(hookset.resourcebase_detail_url(resourcebase_obj)) + + message = resourcebase_obj.id + + try: + # Keywords from THESAURUS management + # Rewritten to work with updated autocomplete + if not tkeywords_form.is_valid(): + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) + + thesaurus_setting = getattr(settings, "THESAURUS", None) + if thesaurus_setting: + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) + resourcebase_obj.tkeywords.set(tkeywords_data) + elif Thesaurus.objects.all().exists(): + fields = tkeywords_form.cleaned_data + resourcebase_obj.tkeywords.set(tkeywords_form.cleanx(fields)) + + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + if "group" in resourcebase_form.changed_data: + vals["group"] = resourcebase_form.cleaned_data.get("group") + if any([x in resourcebase_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = resourcebase_form.cleaned_data.get("is_approved", resourcebase_obj.is_approved) + vals["is_published"] = resourcebase_form.cleaned_data.get("is_published", resourcebase_obj.is_published) + else: + vals.pop("is_approved", None) + vals.pop("is_published", None) + + resource_manager.update( + resourcebase_obj.uuid, + instance=resourcebase_obj, + keywords=new_keywords, + regions=new_regions, + notify=True, + vals=vals, + extra_metadata=json.loads(extra_metadata), + ) + + resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False) + + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not resourcebase_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **resourcebase_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), + } + logger.error(f"resourcebase Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + # - POST Request Ends here - + + # define contact role forms + contact_role_forms_context = {} + for role in resourcebase_obj.get_multivalue_role_property_names(): + resourcebase_form.fields[role].initial = [p.username for p in resourcebase_obj.__getattribute__(role)] + role_form = ProfileForm(prefix=role) + role_form.hidden = True + contact_role_forms_context[f"{role}_form"] = role_form + + metadata_author_groups = get_user_visible_groups(request.user) + + if not AdvancedSecurityWorkflowManager.is_allowed_to_publish(request.user, resourcebase_obj): + resourcebase_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) + if not AdvancedSecurityWorkflowManager.is_allowed_to_approve(request.user, resourcebase_obj): + resourcebase_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) + + register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj) + return render( + request, + template, + context={ + "resource": resourcebase_obj, + "resourcebase": resourcebase_obj, + "panel_template": panel_template, + "custom_metadata": custom_metadata, + "resourcebase_form": resourcebase_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + **contact_role_forms_context, + "UI_ROLES_IN_TOGGLE_VIEW": resourcebase_obj.get_ui_toggled_role_property_names(), + }, + ) + + +@login_required +def resourcebase_metadata_advanced(request, resourcebaseid): + return resourcebase_metadata(request, resourcebaseid, template="base/base_metadata_advanced.html") diff --git a/geonode/client/hooksets.py b/geonode/client/hooksets.py index 84fcc5ec4b3..4745a4601fd 100644 --- a/geonode/client/hooksets.py +++ b/geonode/client/hooksets.py @@ -119,6 +119,9 @@ def geoapp_list_url(self): def geoapp_detail_url(self, geoapp): return NotImplemented + def resourcebase_embed_template(self, context=None): + return None + # Documents def document_list_url(self): return NotImplemented @@ -139,3 +142,6 @@ def metadata_update_redirect(self, url, request=None): if "metadata_uri" in url: return url.replace("/metadata_uri", "") return url.replace("/metadata", "") + + def get_absolute_url(self, context=None): + return None diff --git a/geonode/client/templatetags/client_lib_tags.py b/geonode/client/templatetags/client_lib_tags.py index 3ce659a32bf..2ec9837b06f 100644 --- a/geonode/client/templatetags/client_lib_tags.py +++ b/geonode/client/templatetags/client_lib_tags.py @@ -187,6 +187,11 @@ def render(self, context): elif self.tag_name == "get_geoapp_download": t = context.template.engine.get_template(hookset.geoapp_download_template(context=context)) + # 3DTILES + + if self.tag_name == "get_resourcebase_embed": + t = context.template.engine.get_template(hookset.resourcebase_embed_template(context=context)) + if t: return t.render(context) else: @@ -225,3 +230,5 @@ def do_get_client_library_template(parser, token): register.tag("get_geoapp_update", do_get_client_library_template) register.tag("get_geoapp_embed", do_get_client_library_template) register.tag("get_geoapp_download", do_get_client_library_template) + +register.tag("get_resourcebase_embed", do_get_client_library_template) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index d8fe1ca2395..d68ca3a09b8 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -28,6 +28,7 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode import settings +from geonode.assets.utils import create_asset_and_link from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter from geonode.base.api.mixins import AdvertisedListMixin from geonode.base.api.pagination import GeoNodeApiPagination @@ -47,6 +48,7 @@ import logging + logger = logging.getLogger(__name__) @@ -119,16 +121,20 @@ def perform_create(self, serializer): "extension": extension, "resource_type": "document", } - if file: - manager = StorageManager(remote_files={"base_file": file}) - manager.clone_remote_files() - payload["files"] = [manager.get_retrieved_paths().get("base_file")] if doc_url: payload["doc_url"] = doc_url payload["sourcetype"] = enumerations.SOURCE_TYPE_REMOTE resource = serializer.save(**payload) + if file: + manager = StorageManager(remote_files={"base_file": file}) + manager.clone_remote_files() + create_asset_and_link( + resource, self.request.user, [manager.get_retrieved_paths().get("base_file")], clone_files=True + ) + manager.delete_retrieved_paths(force=True) + resource.set_missing_info() resourcebase_post_save(resource.get_real_instance()) resource_manager.set_permissions(None, instance=resource, permissions=None, created=True) @@ -136,6 +142,7 @@ def perform_create(self, serializer): resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) return resource except Exception as e: + logger.error(f"Error creating document {serializer.validated_data}", exc_info=e) if manager: manager.delete_retrieved_paths() raise e diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 25cc5c2b86d..cdd069929a5 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -26,6 +26,7 @@ from django.utils.functional import classproperty from django.utils.translation import gettext_lazy as _ +from geonode.assets.models import Asset from geonode.client.hooks import hookset from geonode.base.models import ResourceBase from geonode.groups.conf import settings as groups_settings @@ -76,6 +77,11 @@ def compact_permission_labels(cls): "owner": _("Owner"), } + @property + def files(self): + asset = Asset.objects.filter(link__resource=self).first() + return asset.location if asset else [] + @property def name(self): if not self.title: diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index e0ed9617354..316bdafdfe8 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -26,6 +26,8 @@ from geonode.celery_app import app from geonode.storage.manager import StorageManager +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset from ..base.models import ResourceBase from .models import Document @@ -90,7 +92,7 @@ def create_document_thumbnail(self, object_id): """ logger.debug(f"Generating thumbnail for document #{object_id}.") - storage_manager = StorageManager() + default_storage_manager = StorageManager() try: document = Document.objects.get(id=object_id) @@ -104,15 +106,24 @@ def create_document_thumbnail(self, object_id): centering = (0.5, 0.5) doc_path = None - if document.files: - doc_path = storage_manager.path(document.files[0]) + + # get asset of the resource + asset = get_default_asset(document) + if not asset and not document.doc_url: + raise Exception("Document has neither an associated Asset nor a link, cannot generate thumbnail") + + if asset: + handler = asset_handler_registry.get_handler(asset) + asset_storage_manager = handler.get_storage_manager(asset) + doc_path = asset_storage_manager.path(asset.location[0]) elif document.doc_url: doc_path = document.doc_url remove_tmp_file = True + asset_storage_manager = default_storage_manager if document.is_image: try: - image_file = storage_manager.open(doc_path) + image_file = asset_storage_manager.open(doc_path) except Exception as e: logger.debug(f"Could not generate thumbnail from remote document {document.doc_url}: {e}") @@ -129,12 +140,12 @@ def create_document_thumbnail(self, object_id): if image_file is not None: image_file.close() if remove_tmp_file: - storage_manager.delete(doc_path) + default_storage_manager.delete(doc_path) elif doc_renderer.supports(doc_path): # in case it's a remote document we want to retrieve it first if document.doc_url: - doc_path = storage_manager.open(doc_path).name + doc_path = default_storage_manager.open(doc_path).name remove_tmp_file = True try: thumbnail_content = doc_renderer.render(doc_path) @@ -145,7 +156,7 @@ def create_document_thumbnail(self, object_id): print(e) finally: if remove_tmp_file: - storage_manager.delete(doc_path) + default_storage_manager.delete(doc_path) if not thumbnail_content: logger.warning(f"Thumbnail for document #{object_id} empty.") ResourceBase.objects.filter(id=document.id).update(thumbnail_url=None) diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index e8adaa83c6e..69a82b78903 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -42,6 +42,8 @@ from guardian.shortcuts import get_anonymous_user +from geonode.assets.utils import create_asset_and_link +from geonode.base.forms import LinkedResourceForm from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.compat import ensure_string @@ -58,7 +60,9 @@ from geonode.upload.api.exceptions import FileUploadLimitException from .forms import DocumentCreateForm -from ..base.forms import LinkedResourceForm + + +TEST_GIF = os.path.join(os.path.dirname(__file__), "tests/data/img.gif") class DocumentsTest(GeoNodeBaseTestSupport): @@ -113,10 +117,10 @@ def test_document_mimetypes_rendering(self): def test_create_document_with_no_rel(self, thumb): """Tests the creation of a document with no relations""" thumb.return_value = True - f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - c = Document.objects.create(files=f, owner=superuser, title="theimg") + c = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(c, superuser, [TEST_GIF]) c.set_default_permissions() self.assertEqual(Document.objects.get(pk=c.id).title, "theimg") @@ -412,11 +416,11 @@ def test_ajax_document_permissions(self, create_thumb): """Verify that the ajax_document_permissions view is behaving as expected""" create_thumb.return_value = True # Setup some document names to work with - f = [f"{settings.MEDIA_ROOT}/img.gif"] - superuser = get_user_model().objects.get(pk=2) document = resource_manager.create( - None, resource_type=Document, defaults=dict(files=f, owner=superuser, title="theimg", is_approved=True) + None, + resource_type=Document, + defaults=dict(files=[TEST_GIF], owner=superuser, title="theimg", is_approved=True), ) document_id = document.id invalid_document_id = 20 @@ -630,10 +634,10 @@ def setUp(self): def test_create_document_with_links(self): """Tests the creation of document links.""" - f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - d = Document.objects.create(files=f, owner=superuser, title="theimg") + d = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(d, superuser, [TEST_GIF]) self.assertEqual(Document.objects.get(pk=d.id).title, "theimg") @@ -679,11 +683,10 @@ def setUp(self): self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True) self.not_admin.set_password("very-secret") self.not_admin.save() - self.files = [f"{settings.MEDIA_ROOT}/img.gif"] self.test_doc = resource_manager.create( None, resource_type=Document, - defaults=dict(files=self.files, owner=self.not_admin, title="test", is_approved=True), + defaults=dict(files=[TEST_GIF], owner=self.not_admin, title="test", is_approved=True), ) self.perm_spec = {"users": {"AnonymousUser": []}} self.doc_link_url = reverse("document_link", args=(self.test_doc.pk,)) @@ -808,7 +811,7 @@ def test_document_link_with_permissions(self): # Access resource with user logged-in self.client.login(username=self.not_admin.username, password="very-secret") response = self.client.get(self.doc_link_url) - self.assertEqual(response.status_code, 404) + self.assertEqual(response.status_code, 200) # test document link with external url doc = resource_manager.create( None, diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py index 63facbbaf09..8f5ec4ad619 100644 --- a/geonode/documents/utils.py +++ b/geonode/documents/utils.py @@ -23,6 +23,9 @@ # Standard Modules import os import logging + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset from geonode.storage.manager import storage_manager # Django functionality @@ -31,7 +34,6 @@ from django.template import loader from django.utils.translation import gettext_lazy as _ from django.utils.text import slugify -from django_downloadview.response import DownloadResponse # Geonode functionality from geonode.documents.models import Document @@ -78,10 +80,6 @@ def get_download_response(request, docid, attachment=False): register_event(request, EventType.EVENT_DOWNLOAD, document) filename = slugify(os.path.splitext(os.path.basename(document.title))[0]) - if document.files and storage_manager.exists(document.files[0]): - return DownloadResponse( - storage_manager.open(document.files[0]).file, - basename=f"{filename}.{document.extension}", - attachment=attachment, - ) - return HttpResponse("File is not available", status=404) + asset = get_default_asset(document) + asset_handler = asset_handler_registry.get_handler(asset) + return asset_handler.get_download_handler(asset).create_response(asset, attachment, basename=filename) diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 545fd715647..411f80a4bd0 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -33,8 +33,10 @@ from django.views.generic.edit import CreateView, UpdateView from django.http import HttpResponse, HttpResponseRedirect, Http404 from django.core.exceptions import PermissionDenied, ObjectDoesNotExist -from geonode.base.api.exceptions import geonode_exception_handler +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset +from geonode.base.api.exceptions import geonode_exception_handler from geonode.client.hooks import hookset from geonode.utils import mkdtemp, resolve_object from geonode.base.views import batch_modify @@ -169,12 +171,21 @@ def form_valid(self, form): owner=self.request.user, doc_url=doc_form.pop("doc_url", None), title=doc_form.pop("title", file.name), + description=doc_form.pop("abstract", None), extension=doc_form.pop("extension", None), + link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES + data_title=doc_form.pop("title", file.name), + data_type=doc_form.pop("extension", None), files=[storage_path], ), ) - if tempdir != os.path.dirname(storage_path): - shutil.rmtree(tempdir, ignore_errors=True) + + # Removing the temp file + # TODO: creating a file and then cloning it as an Asset may be slow: we may want to + # create the file directly in the asset dir or to move it + logger.info(f"Removing document temp dir {tempdir}") + shutil.rmtree(tempdir, ignore_errors=True) + else: self.object = resource_manager.create( None, @@ -278,11 +289,17 @@ def form_valid(self, form): if file: tempdir = mkdtemp() dirname = os.path.basename(tempdir) - filepath = storage_manager.save(f"{dirname}/{file.name}", file) + filepath = storage_manager.save(os.path.join(dirname, file.name), file) storage_path = storage_manager.path(filepath) self.object = resource_manager.update( - self.object.uuid, instance=self.object, vals=dict(owner=self.request.user, files=[storage_path]) + self.object.uuid, instance=self.object, vals=dict(owner=self.request.user) ) + + # replace data in existing asset + asset = get_default_asset(self.object, link_type="uploaded") + if asset: + asset_handler_registry.get_handler(asset).replace_data(asset, [storage_path]) + if tempdir != os.path.dirname(storage_path): shutil.rmtree(tempdir, ignore_errors=True) diff --git a/geonode/geoserver/manager.py b/geonode/geoserver/manager.py index a88b641b71f..e3cc4a69378 100644 --- a/geonode/geoserver/manager.py +++ b/geonode/geoserver/manager.py @@ -290,7 +290,6 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non _to_update = { "name": _name, "title": instance.title or _gs_import_session_info.dataset_name, - "files": kwargs.get("files", None), "workspace": _gs_import_session_info.workspace, "alternate": _alternate, "typename": _alternate, diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py index 85672984b00..7d2b9ac2155 100644 --- a/geonode/geoserver/tests/test_manager.py +++ b/geonode/geoserver/tests/test_manager.py @@ -19,6 +19,7 @@ import os import base64 import shutil +from django.test import override_settings import gisdata import requests @@ -54,8 +55,14 @@ def tearDown(self) -> None: return super().tearDown() @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @override_settings(ASYNC_SIGNALS=False, FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777, FILE_UPLOAD_PERMISSIONS=0o7777) def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self): layer = Dataset.objects.get(name=self.sut.name) + gs_layer = self.cat.get_layer("san_andres_y_providencia_water") + if gs_layer is None: + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="create" + ) _gs_import_session_info = self.geoserver_manager._execute_resource_import( layer, list(self.files_as_dict.values()), self.user, action_type="append" ) diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py index 53b4bf7af76..b7102116c9e 100644 --- a/geonode/layers/utils.py +++ b/geonode/layers/utils.py @@ -507,7 +507,7 @@ def get_uuid_handler(): dataset_download_handler_list = [] -def get_dataset_download_handlers(): +def get_download_handlers(): if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None): dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0])) diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index 6ad3febe547..5b7beafc906 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -130,6 +130,7 @@ def perform_create(self, serializer): create_action_perfomed=True, additional_data=post_creation_data, ) + # Handle thumbnail generation resource_manager.set_thumbnail(instance.uuid, instance=instance, overwrite=False) diff --git a/geonode/proxy/templatetags/proxy_lib_tags.py b/geonode/proxy/templatetags/proxy_lib_tags.py index 1135d12bd86..5993d7ef67e 100644 --- a/geonode/proxy/templatetags/proxy_lib_tags.py +++ b/geonode/proxy/templatetags/proxy_lib_tags.py @@ -17,6 +17,7 @@ # ######################################################################### +from geonode.assets.utils import get_default_asset from geonode.base.models import ResourceBase import traceback @@ -52,7 +53,10 @@ def original_link_available(context, resourceid, url): dataset_files = [] if isinstance(instance, ResourceBase): try: - for file in instance.files: + asset_obj = get_default_asset(instance) + # Copy all Dataset related files into a temporary folder + files = asset_obj.location if asset_obj else [] + for file in files: dataset_files.append(file) if not storage_manager.exists(file): return False diff --git a/geonode/proxy/tests.py b/geonode/proxy/tests.py index 2f094488ee0..20a0b091b29 100644 --- a/geonode/proxy/tests.py +++ b/geonode/proxy/tests.py @@ -30,6 +30,7 @@ from urllib.parse import urljoin from django.conf import settings +from geonode.assets.utils import create_asset_and_link from geonode.proxy.templatetags.proxy_lib_tags import original_link_available from django.test.client import RequestFactory from django.core.files.uploadedfile import SimpleUploadedFile @@ -308,12 +309,15 @@ def test_download_url_with_existing_files(self, fopen, fexists): fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc") dataset = Dataset.objects.all().first() - dataset.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) dataset.save() @@ -331,6 +335,9 @@ def test_download_url_with_existing_files(self, fopen, fexists): self.assertEqual("application/zip", response.headers.get("Content-Type")) self.assertEqual('attachment; filename="CA.zip"', response.headers.get("Content-Disposition")) + link.delete() + asset.delete() + @patch("geonode.storage.manager.storage_manager.exists") @patch("geonode.storage.manager.storage_manager.open") @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -339,12 +346,15 @@ def test_download_files(self, fopen, fexists): fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc") dataset = Dataset.objects.all().first() - dataset.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) dataset.save() @@ -368,6 +378,9 @@ def test_download_files(self, fopen, fexists): self.assertIn(".shx", "".join(zip_files)) self.assertIn(".prj", "".join(zip_files)) + link.delete() + asset.delete() + class OWSApiTestCase(GeoNodeBaseTestSupport): def setUp(self): @@ -420,16 +433,23 @@ def test_should_return_true_if_files_are_available(self, fexists): assert upload - self.resource.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + self.resource, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) + self.resource.save() self.resource.refresh_from_db() actual = original_link_available(self.context, self.resource.resourcebase_ptr_id, self.url) self.assertTrue(actual) + + link.delete() + asset.delete() diff --git a/geonode/proxy/views.py b/geonode/proxy/views.py index b7bca6ee5b9..20fe0158ca5 100644 --- a/geonode/proxy/views.py +++ b/geonode/proxy/views.py @@ -55,6 +55,7 @@ from geonode.base import register_event from geonode.base.auth import get_auth_user, get_token_from_auth_header from geonode.geoserver.helpers import ogc_server_settings +from geonode.assets.utils import get_default_asset from .utils import proxy_urls_registry @@ -245,8 +246,9 @@ def download(request, resourceid, sender=Dataset): dataset_files = [] file_list = [] # Store file info to be returned try: - files = instance.resourcebase_ptr.files + asset_obj = get_default_asset(instance) # Copy all Dataset related files into a temporary folder + files = asset_obj.location if asset_obj else [] for file_path in files: if storage_manager.exists(file_path): dataset_files.append(file_path) diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py index d4c94a1bc66..eefab451b4a 100644 --- a/geonode/resource/manager.py +++ b/geonode/resource/manager.py @@ -21,6 +21,7 @@ import copy import typing import logging +import itertools from uuid import uuid1, uuid4 from abc import ABCMeta, abstractmethod @@ -38,6 +39,8 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist, ValidationError, FieldDoesNotExist + +from geonode.base.models import ResourceBase, LinkedResource from geonode.thumbs.thumbnails import _generate_thumbnail_name from geonode.documents.tasks import create_document_thumbnail from geonode.security.permissions import PermSpecCompact, DATA_STYLABLE_RESOURCES_SUBTYPES @@ -45,9 +48,9 @@ from . import settings as rm_settings from .utils import update_resource, resourcebase_post_save +from geonode.assets.utils import create_asset_and_link_dict, rollback_asset_and_link, copy_assets_and_links, create_link from ..base import enumerations -from ..base.models import ResourceBase, LinkedResource from ..security.utils import AdvancedSecurityWorkflowManager from ..layers.metadata import parse_metadata from ..documents.models import Document @@ -313,20 +316,35 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de if resource_type.objects.filter(uuid=uuid).exists(): return resource_type.objects.filter(uuid=uuid).get() uuid = uuid or str(uuid4()) - _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=defaults) + resource_dict = { # TODO: cleanup params and dicts + k: v + for k, v in defaults.items() + if k not in ("data_title", "data_type", "description", "files", "link_type", "extension", "asset") + } + _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=resource_dict) if _resource and _created: _resource.set_processing_state(enumerations.STATE_RUNNING) try: + # if files exist: create an Asset out of them and link it to the Resource + asset, link = (None, None) # safe init in case of exception + if defaults.get("files", None): + logger.debug(f"Found files when creating resource {_resource}: {defaults['files']}") + asset, link = create_asset_and_link_dict(_resource, defaults, clone_files=True) + elif defaults.get("asset", None): + logger.debug(f"Found asset when creating resource {_resource}: {defaults['asset']}") + link = create_link(_resource, **defaults) + with transaction.atomic(): _resource.set_missing_info() _resource = self._concrete_resource_manager.create( - uuid, resource_type=resource_type, defaults=defaults + uuid, resource_type=resource_type, defaults=resource_dict ) _resource.save() resourcebase_post_save(_resource.get_real_instance()) _resource.set_processing_state(enumerations.STATE_PROCESSED) except Exception as e: logger.exception(e) + rollback_asset_and_link(asset, link) # we are not removing the Asset passed in defaults self.delete(_resource.uuid, instance=_resource) raise e return _resource @@ -440,19 +458,19 @@ def ingest( ) -> ResourceBase: instance = None to_update = defaults.copy() - if "files" in to_update: - to_update.pop("files") + to_update_with_files = {**to_update, **{"files": files}} try: with transaction.atomic(): if resource_type == Document: if "name" in to_update: to_update.pop("name") - if files: - to_update["files"] = storage_manager.copy_files_list(files) - instance = self.create(uuid, resource_type=Document, defaults=to_update) + instance = self.create(uuid, resource_type=Document, defaults=to_update_with_files) elif resource_type == Dataset: if files: - instance = self.create(uuid, resource_type=Dataset, defaults=to_update) + instance = self.create(uuid, resource_type=Dataset, defaults=to_update_with_files) + else: + logger.warning(f"Will not create a Dataset without any file. Values: {defaults}") + if instance: instance = self._concrete_resource_manager.ingest( storage_manager.copy_files_list(files), @@ -523,11 +541,15 @@ def copy( _maplayer.pk = _maplayer.id = None _maplayer.map = _resource.get_real_instance() _maplayer.save() + + assets_and_links = copy_assets_and_links(instance, target=_resource) + # we're just merging all the files together: it won't work once we have multiple assets per resource + # TODO: get the files from the proper Asset, or make the _concrete_resource_manager.copy use assets to_update = {} - try: - to_update = storage_manager.copy(_resource).copy() - except Exception as e: - logger.exception(e) + + files = list(itertools.chain.from_iterable([asset.location for asset, _ in assets_and_links])) + if files: + to_update = {"files": files} _resource = self._concrete_resource_manager.copy(instance, uuid=_resource.uuid, defaults=to_update) diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py index 4b25308afbe..a1850861673 100644 --- a/geonode/resource/tests.py +++ b/geonode/resource/tests.py @@ -148,7 +148,15 @@ def _copy_assert_resource(res, title): # copy with documents res = self.rm.ingest( - dt_files, resource_type=Document, defaults={"title": "relief_san_andres", "owner": self.user} + dt_files, + resource_type=Document, + defaults={ + "title": "relief_san_andres", + "owner": self.user, + "extension": "tif", + "data_title": "relief_san_andres", + "data_type": "tif", + }, ) self.assertTrue(isinstance(res, Document)) _copy_assert_resource(res, "Testing Document 2") @@ -157,7 +165,12 @@ def _copy_assert_resource(res, title): res = self.rm.ingest( dt_files, resource_type=Dataset, - defaults={"owner": self.user, "title": "Testing Dataset", "files": dt_files}, + defaults={ + "owner": self.user, + "title": "Testing Dataset", + "data_title": "relief_san_andres", + "data_type": "tif", + }, ) self.assertTrue(isinstance(res, Dataset)) _copy_assert_resource(res, "Testing Dataset 2") diff --git a/geonode/resource/utils.py b/geonode/resource/utils.py index f0e8e2201e5..78e215173e8 100644 --- a/geonode/resource/utils.py +++ b/geonode/resource/utils.py @@ -29,9 +29,11 @@ from django.utils import timezone from django.core.exceptions import FieldDoesNotExist from django.utils.translation import gettext_lazy as _ -from geonode.utils import OGC_Servers_Handler from django.utils.module_loading import import_string +from geonode.assets.utils import get_default_asset +from geonode.utils import OGC_Servers_Handler + from ..base import enumerations from ..base.models import ( ExtraMetadata, @@ -241,10 +243,13 @@ def update_resource( ] to_update.update(defaults) + resource_dict = { # TODO: cleanup params and dicts + k: v for k, v in to_update.items() if k not in ("data_title", "data_type", "description", "files", "link_type") + } try: - instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**to_update) + instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**resource_dict) except Exception as e: - logger.error(f"{e} - {to_update}") + logger.error(f"{e} - {resource_dict}") raise # Check for "remote services" availability @@ -322,9 +327,9 @@ def get_alternate_name(instance): def document_post_save(instance, *args, **kwargs): instance.csw_type = "document" - - if instance.files: - _, extension = os.path.splitext(os.path.basename(instance.files[0])) + asset = get_default_asset(instance) + if asset: + _, extension = os.path.splitext(os.path.basename(asset.location[0])) instance.extension = extension[1:] doc_type_map = DOCUMENT_TYPE_MAP doc_type_map.update(getattr(settings, "DOCUMENT_TYPE_MAP", {})) @@ -344,7 +349,7 @@ def document_post_save(instance, *args, **kwargs): mime = mime_type_map.get(ext, "text/plain") url = None - if instance.id and instance.files: + if instance.id and asset: name = "Hosted Document" site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL url = f"{site_url}{reverse('document_download', args=(instance.id,))}" @@ -455,8 +460,10 @@ def resourcebase_post_save(instance, *args, **kwargs): if hasattr(instance, "abstract") and not getattr(instance, "abstract", None): instance.abstract = _("No abstract provided") if hasattr(instance, "title") and not getattr(instance, "title", None) or getattr(instance, "title", "") == "": - if isinstance(instance, Document) and instance.files: - instance.title = os.path.basename(instance.files[0]) + asset = get_default_asset(instance) + files = asset.location if asset else [] + if isinstance(instance, Document) and files: + instance.title = os.path.basename(files[0]) if hasattr(instance, "name") and getattr(instance, "name", None): instance.title = instance.name if ( diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 5ae00733cf4..ce9b40a507f 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -21,9 +21,11 @@ import base64 import logging import uuid +import os import requests import importlib import mock +import gisdata from requests.auth import HTTPBasicAuth from tastypie.test import ResourceTestCaseMixin @@ -40,7 +42,9 @@ from guardian.shortcuts import assign_perm, get_anonymous_user from geonode import geoserver -from geonode.geoserver.helpers import geofence, gf_utils +from geonode.geoserver.helpers import geofence, gf_utils, gs_catalog +from geonode.geoserver.manager import GeoServerResourceManager +from geonode.layers.utils import get_files from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.documents.models import Document @@ -742,8 +746,19 @@ def test_perm_specs_synchronization(self): @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_permissions(self): # Test permissions on a layer + files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_poi.shp") + files_as_dict, self.tmpdir = get_files(files) + bobby = get_user_model().objects.get(username="bobby") - layer = create_single_dataset("san_andres_y_providencia_poi") + layer = create_single_dataset( + "san_andres_y_providencia_poi", + { + "owner": self.user, + "title": "Testing Dataset", + "data_title": "relief_san_andres", + "data_type": "tif", + }, + ) layer = resource_manager.update( layer.uuid, instance=layer, notify=False, vals=dict(owner=bobby, workspace=settings.DEFAULT_WORKSPACE) ) @@ -774,6 +789,15 @@ def test_dataset_permissions(self): perm_spec = {"users": {"AnonymousUser": []}, "groups": []} layer.set_permissions(perm_spec) + gs_layer = gs_catalog.get_layer("3Asan_andres_y_providencia_poi") + if gs_layer is None: + GeoServerResourceManager()._execute_resource_import( + layer, + list(files_as_dict.values()), + get_user_model().objects.get(username="admin"), + action_type="create", + ) + url = ( f"{settings.GEOSERVER_LOCATION}ows?" "LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES=" @@ -786,7 +810,8 @@ def test_dataset_permissions(self): # test view_resourcebase permission on anonymous user response = requests.get(url) - self.assertTrue(response.status_code, 404) + self.assertEqual(response.status_code, 200) + self.assertTrue(b"Could not find layer" in response.content) self.assertEqual(response.headers.get("Content-Type"), "application/vnd.ogc.se_xml;charset=UTF-8") # test WMS with authenticated user that has access to the Dataset @@ -796,7 +821,7 @@ def test_dataset_permissions(self): username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"] ), ) - self.assertTrue(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(response.headers.get("Content-Type"), "image/png") # test WMS with authenticated user that has no view_resourcebase: diff --git a/geonode/settings.py b/geonode/settings.py index 84333dc7ef2..098dd86ba79 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -300,6 +300,11 @@ # Example: "/home/media/media.lawrence.com/apps/" STATIC_ROOT = os.getenv("STATIC_ROOT", os.path.join(PROJECT_ROOT, "static_root")) +# Absolute path to the directory that hold assets files +# This dir should not be made publicly accessible by nginx, since its content may be private +# Using a sibling of MEDIA_ROOT as default +ASSETS_ROOT = os.getenv("ASSETS_ROOT", os.path.join(os.path.dirname(MEDIA_ROOT.rstrip("/")), "assets_data")) + # Cache Bustin Settings: enable WhiteNoise compression and caching support # ref: http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support CACHE_BUSTING_STATIC_ENABLED = ast.literal_eval(os.environ.get("CACHE_BUSTING_STATIC_ENABLED", "False")) @@ -701,7 +706,7 @@ }, "loggers": { "django": { - "level": "ERROR", + "level": "WARN", }, "geonode": { "level": "WARN", @@ -2340,6 +2345,7 @@ def get_geonode_catalogue_service(): 'importer.handlers.geotiff.handler.GeoTiffFileHandler',\ 'importer.handlers.xml.handler.XMLFileHandler',\ 'importer.handlers.sld.handler.SLDFileHandler',\ + 'importer.handlers.tiles3d.handler.Tiles3DFileHandler',\ ]", ) ) @@ -2365,3 +2371,10 @@ def get_geonode_catalogue_service(): AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS = ast.literal_eval( os.getenv("AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS", "True") ) + +DEFAULT_ASSET_HANDLER = "geonode.assets.local.LocalAssetHandler" +ASSET_HANDLERS = [ + DEFAULT_ASSET_HANDLER, +] +INSTALLED_APPS += ("geonode.assets",) +GEONODE_APPS += ("geonode.assets",) diff --git a/geonode/storage/data_retriever.py b/geonode/storage/data_retriever.py index bc23997a8a2..f7178937228 100644 --- a/geonode/storage/data_retriever.py +++ b/geonode/storage/data_retriever.py @@ -151,10 +151,12 @@ def __init__(self, files, tranfer_at_creation=False): if tranfer_at_creation: self.transfer_remote_files() - def transfer_remote_files(self): + def transfer_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True): from geonode.utils import mkdtemp - self.temporary_folder = mkdtemp() + self.temporary_folder = cloning_directory or settings.MEDIA_ROOT + if create_tempdir: + self.temporary_folder = mkdtemp(cloning_directory or settings.MEDIA_ROOT, prefix=prefix) for name, data_item_retriever in self.data_items.items(): file_path = data_item_retriever.transfer_remote_file(self.temporary_folder) self.file_paths[name] = Path(file_path) @@ -172,10 +174,12 @@ def transfer_remote_files(self): os.chmod(self.temporary_folder, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) return self.file_paths - def get_paths(self, allow_transfer=False): + def get_paths(self, allow_transfer=False, cloning_directory=None, prefix=None, create_tempdir=True): if not self.file_paths: if allow_transfer: - self.transfer_remote_files() + self.transfer_remote_files( + cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir + ) else: raise DataRetrieverExcepion(detail="You can't retrieve paths without clone file first!") return self.file_paths.copy() diff --git a/geonode/storage/manager.py b/geonode/storage/manager.py index 2966fc6924e..4d1cbbb0133 100644 --- a/geonode/storage/manager.py +++ b/geonode/storage/manager.py @@ -124,8 +124,8 @@ class StorageManager(StorageManagerInterface): treat as a file_system file """ - def __init__(self, remote_files: Mapping = {}): - self._concrete_storage_manager = self._get_concrete_manager() + def __init__(self, remote_files: Mapping = {}, concrete_storage_manager=None): + self._concrete_storage_manager = concrete_storage_manager or self._get_concrete_manager() self.data_retriever = DataRetriever(remote_files, tranfer_at_creation=False) def _get_concrete_manager(self): @@ -174,18 +174,19 @@ def replace(self, resource, files: Union[list, BinaryIO]): updated_files["files"] = [self.replace_single_file(resource.files[0], files)] return updated_files - def copy(self, resource): - updated_files = {} - if len(resource.files): - updated_files["files"] = self.copy_files_list(resource.files) - return updated_files + def copy(self, resource, target=None): + raise Exception("This is not the copy you're looking for") + # updated_files = {} + # if len(resource.files): + # updated_files["files"] = self.copy_files_list(resource.files) + # return updated_files - def copy_files_list(self, files: List[str]): + def copy_files_list(self, files: List[str], dir=settings.MEDIA_ROOT, dir_prefix=None, dir_suffix=None): from geonode.utils import mkdtemp out = [] random_suffix = f"{uuid1().hex[:8]}" - new_path = mkdtemp() + new_path = mkdtemp(dir=dir, prefix=dir_prefix, suffix=dir_suffix) if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: # value is always set by default as None @@ -242,11 +243,13 @@ def replace_single_file(self, old_file: str, new_file: BinaryIO, prefix: str = N def generate_filename(self, filename): return self._concrete_storage_manager.generate_filename(filename) - def clone_remote_files(self) -> Mapping: + def clone_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True) -> Mapping: """ Using the data retriever object clone the remote path into a local temporary storage """ - return self.data_retriever.get_paths(allow_transfer=True) + return self.data_retriever.get_paths( + allow_transfer=True, cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir + ) def get_retrieved_paths(self) -> Mapping: """ @@ -266,8 +269,8 @@ def delete_retrieved_paths(self, force=False) -> None: class DefaultStorageManager(StorageManagerInterface): - def __init__(self): - self._fsm = FileSystemStorage() + def __init__(self, **kwargs): + self._fsm = FileSystemStorage(**kwargs) def _get_concrete_manager(self): return DefaultStorageManager() diff --git a/geonode/storage/tests.py b/geonode/storage/tests.py index 9a96186ab3e..43adc794ffa 100644 --- a/geonode/storage/tests.py +++ b/geonode/storage/tests.py @@ -401,22 +401,6 @@ def test_storage_manager_replace_single_file(self, path, strg): output = self.sut().replace(dataset, new_file) self.assertListEqual([expected], output["files"]) - @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777) - @override_settings(FILE_UPLOAD_PERMISSIONS=0o777) - def test_storage_manager_copy(self): - """ - Test that the copy works as expected and the permissions are corerct - """ - dataset = create_single_dataset(name="test_copy") - dataset.files = [os.path.join(f"{self.project_root}", "tests/data/test_sld.sld")] - dataset.save() - output = self.sut().copy(dataset) - - self.assertTrue(os.path.exists(output.get("files")[0])) - self.assertEqual(os.stat(os.path.exists(output.get("files")[0])).st_mode, 8592) - os.remove(output.get("files")[0]) - self.assertFalse(os.path.exists(output.get("files")[0])) - class TestDataRetriever(TestCase): @classmethod diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py index c607799d07d..75a4dee41fc 100644 --- a/geonode/upload/api/tests.py +++ b/geonode/upload/api/tests.py @@ -231,6 +231,8 @@ def live_upload_file(self, _file): return response, response.content @override_settings(CELERY_TASK_ALWAYS_EAGER=True) + @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777) + @override_settings(FILE_UPLOAD_PERMISSIONS=0o777) def test_rest_uploads(self): """ Ensure we can access the Local Server Uploads list. @@ -255,12 +257,21 @@ def test_rest_uploads(self): self.assertEqual(len(response.data["uploads"]), 0) logger.debug(response.data) except Exception: - if resp.json().get("errors"): - layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + json = resp.json() + if json.get("errors"): + logger.error(f"Error in upload: {json}") + try: + layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0] + except IndexError as e: + logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e) + # TODO: make sure the _cleanup_layer will use the proper layer name + self.skipTest("Error with GeoServer") finally: self._cleanup_layer(layer_name) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) + @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777) + @override_settings(FILE_UPLOAD_PERMISSIONS=0o777) def test_rest_uploads_non_interactive(self): """ Ensure we can access the Local Server Uploads list. @@ -276,9 +287,16 @@ def test_rest_uploads_non_interactive(self): exec_id = data.get("execution_id", None) _exec = ExecutionRequest.objects.get(exec_id=exec_id) self.assertEqual(_exec.status, "finished") - except Exception: - if resp.json().get("errors"): - layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + except Exception as e: + json = resp.json() + logger.warning(f"Error with GeoServer {json}: {e}", exc_info=e) + if json.get("errors"): + try: + layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0] + except IndexError as e: + logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e) + # TODO: make sure the _cleanup_layer will use the proper layer name + self.skipTest("Error with GeoServer") finally: self._cleanup_layer(layer_name) diff --git a/geonode/urls.py b/geonode/urls.py index ccab35e950f..99b1e5ebaf9 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -69,6 +69,7 @@ urlpatterns += [ # ResourceBase views re_path(r"^base/", include("geonode.base.urls")), + re_path(r"^resources/", include("geonode.base.base_urls")), # Dataset views re_path(r"^datasets/", include("geonode.layers.urls")), # Remote Services views @@ -127,6 +128,7 @@ re_path(r"^api/v2/", include("geonode.management_commands_http.urls")), re_path(r"^api/v2/api-auth/", include("rest_framework.urls", namespace="geonode_rest_framework")), re_path(r"^api/v2/", include("geonode.facets.urls")), + re_path(r"^api/v2/", include("geonode.assets.urls")), re_path(r"", include(api.urls)), ] diff --git a/geonode/utils.py b/geonode/utils.py index c93b61d7cf7..9268feb51da 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -281,13 +281,13 @@ def all(self): return [self[alias] for alias in self] -def mkdtemp(dir=settings.MEDIA_ROOT): +def mkdtemp(dir=settings.MEDIA_ROOT, prefix=None, suffix=None): if not os.path.exists(dir): os.makedirs(dir, exist_ok=True) tempdir = None while not tempdir: try: - tempdir = tempfile.mkdtemp(dir=dir) + tempdir = tempfile.mkdtemp(dir=dir, prefix=prefix, suffix=suffix) if os.path.exists(tempdir) and os.path.isdir(tempdir): if os.listdir(tempdir): raise Exception("Directory is not empty") diff --git a/tasks.py b/tasks.py index b19fa5f7973..82233ee642d 100755 --- a/tasks.py +++ b/tasks.py @@ -341,8 +341,9 @@ def statics(ctx): try: static_root = os.environ.get("STATIC_ROOT", "/mnt/volumes/statics/static/") media_root = os.environ.get("MEDIA_ROOT", "/mnt/volumes/statics/uploaded/") + assets_root = os.environ.get("ASSETS_ROOT", "/mnt/volumes/statics/assets/") - ctx.run(f"mkdir -pv {static_root} {media_root}") + ctx.run(f"mkdir -pv {static_root} {media_root} {assets_root}") ctx.run( f"python manage.py collectstatic --noinput --settings={_localsettings()}", pty=True,