[Fixes #12124] GNIP 100: Assets

.gitignore

@@ -29,6 +29,7 @@ geonode/local_settings.py
 
 # Uploaded files
 geonode/uploaded
+geonode/assets_data
 
 #Testing output
 .coverage

geonode/assets/handlers.py

@@ -12,37 +12,37 @@
 class AssetDownloadHandlerInterface:
 
     def create_response(self, asset: Asset, attachment: bool = False, basename=None, path=None) -> HttpResponse:
         raise NotImplementedError()
 
 
 class AssetHandlerInterface:
 
     def handled_asset_class(self):
         raise NotImplementedError()
 
     def create(self, title, description, type, owner, *args, **kwargs):
         raise NotImplementedError()
 
     def remove_data(self, asset: Asset, **kwargs):
         raise NotImplementedError()
 
     def replace_data(self, asset: Asset, files: list):
         raise NotImplementedError()
 
     def clone(self, asset: Asset) -> Asset:
         """
         Creates a copy in the DB and copies the underlying data as well
         """
         raise NotImplementedError()
 
     def create_link_url(self, asset: Asset) -> str:
         raise NotImplementedError()
 
-    def get_download_handler(self, asset: Asset) -> AssetDownloadHandlerInterface:
+    def get_download_handler(self, asset: Asset = None) -> AssetDownloadHandlerInterface:
         raise NotImplementedError()
 
-    def get_storage_manager(self, asset):
+    def get_storage_manager(self, asset=None):
         raise NotImplementedError()
 
 
 class AssetHandlerRegistry:
@@ -67,7 +67,7 @@
                 break
 
         if self._default_handler is None:
             logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}")
         else:
             logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}")
 

geonode/assets/local.py

@@ -20,17 +20,33 @@
 )
 
 
+class DefaultLocalLinkUrlHandler:
+    def get_link_url(self, asset: LocalAsset):
+        return build_absolute_uri(reverse("assets-link", args=(asset.pk,)))
+
+
+class IndexLocalLinkUrlHandler:
+    def get_link_url(self, asset: LocalAsset):
+        return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + f"/{os.path.basename(asset.location[0])}"
+
+
 class LocalAssetHandler(AssetHandlerInterface):
+
+    link_url_handlers = {"gpkg": IndexLocalLinkUrlHandler()}
+
     @staticmethod
     def handled_asset_class():
         return LocalAsset
 
-    def get_download_handler(self, asset):
+    def get_download_handler(self, asset=None):
         return LocalAssetDownloadHandler()
 
     def get_storage_manager(self, asset):
         return _asset_storage_manager
 
+    def get_link_url_handler(self, asset):
+        return self.link_url_handlers.get(asset.type, None) or DefaultLocalLinkUrlHandler()
+
     def _create_asset_dir(self):
         return os.path.normpath(
             mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
@@ -38,7 +54,7 @@
 
     def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs):
         if not files:
             raise ValueError("File(s) expected")
 
         if clone_files:
             files = self._copy_data(files)
@@ -78,16 +94,16 @@
         new_files = []
         for file in files:
             if os.path.isdir(file):
                 dst = os.path.join(new_path, os.path.basename(file))
                 logging.info(f"Copying into {dst} directory {file}")
                 new_dir = shutil.copytree(file, dst)
                 new_files.append(new_dir)
             elif os.path.isfile(file):
                 logging.info(f"Copying into {new_path} file {os.path.basename(file)}")
                 new_file = shutil.copy2(file, new_path)
                 new_files.append(new_file)
             else:
                 logger.warning(f"Not copying path {file}")
 
         return new_files
 
@@ -98,7 +114,7 @@
         if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
             # value is always set by default as None
             # https://docs.djangoproject.com/en/3.2/ref/settings/#file-upload-directory-permissions
             os.chmod(new_path, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
 
         shutil.copytree(source_dir, new_path, dirs_exist_ok=True)
 
@@ -126,7 +142,7 @@
         return build_absolute_uri(reverse("assets-download", args=(asset.pk,)))
 
     def create_link_url(self, asset) -> str:
-        return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + f"/{os.path.basename(asset.location[0])}"
+        return self.get_link_url_handler(asset).get_link_url(asset)
 
     @classmethod
     def _is_file_managed(cls, file) -> bool:
@@ -155,14 +171,14 @@
     @classmethod
     def _get_managed_dir(cls, asset):
         if not asset.location:
             raise ValueError("Asset does not have any associated file")
 
         assets_root = os.path.normpath(settings.ASSETS_ROOT)
         base_common = None
 
         for file in asset.location:
             if not cls._is_file_managed(file):
                 raise ValueError("Asset is unmanaged")
 
             norm_file = os.path.normpath(file)
             relative = norm_file.removeprefix(assets_root)
@@ -170,16 +186,20 @@
 
             if base_common:
                 if base_common != base:
                     raise ValueError(f"Mismatching base dir in asset files - Asset {asset.pk}")
             else:
                 base_common = base
 
         managed_dir = os.path.join(assets_root, base_common)
+
+        if not os.path.exists(managed_dir):
+            raise ValueError(f"Common dir '{managed_dir}' does not exist - Asset {asset.pk}")
+
         if not os.path.isdir(managed_dir):
             raise ValueError(f"Common dir '{managed_dir}' does not seem to be a directory - Asset {asset.pk}")
 
         if assets_root == managed_dir:  # dunno if this can ever happen, but better safe than sorry
             raise ValueError(f"Common dir '{managed_dir}' matches the whole Assets dir - Asset {asset.pk}")
 
         return managed_dir
 
@@ -190,29 +210,29 @@
         self, asset: LocalAsset, attachment: bool = False, basename: str = None, path: str = None
     ) -> HttpResponse:
         if not asset.location:
             return HttpResponse("Asset does not contain any data", status=500)
 
         if len(asset.location) > 1:
             logger.warning("TODO: Asset contains more than one file. Download needs to be implemented")
 
         file0 = asset.location[0]
         if not path:  # use the file definition
             if not os.path.isfile(file0):
                 logger.warning(f"Default file {file0} not found for asset {asset.id}")
                 return HttpResponse(f"Default file not found for asset {asset.id}", status=400)
             localfile = file0
 
         else:  # a specific file is requested
             if "/../" in path:  # we may want to improve fraudolent request detection
                 logger.warning(f"Tentative path traversal for asset {asset.id}")
                 return HttpResponse(f"File not found for asset {asset.id}", status=400)
 
             if os.path.isfile(file0):
                 dir0 = os.path.dirname(file0)
             elif os.path.isdir(file0):
                 dir0 = file0
             else:
                 return HttpResponse(f"Unexpected internal location '{file0}' for asset {asset.id}", status=500)
 
             localfile = os.path.join(dir0, path)
             logger.debug(f"Requested path {dir0} + {path}")
@@ -230,8 +250,8 @@
                 attachment=attachment,
             )
         else:
             logger.warning(f"Internal file {localfile} not found for asset {asset.id}")
             return HttpResponse(f"Internal file not found for asset {asset.id}", status=404 if path else 500)
 
 
 asset_handler_registry.register(LocalAssetHandler)

geonode/assets/tests/data/one.json

@@ -1,3 +1,3 @@
 {
-  "one": 1
+    "one": 1
 }

geonode/assets/tests/data/three.json

@@ -1,3 +1,3 @@
 {
-  "three": 3
+    "three": 3
 }

geonode/assets/tests/data/two.json

@@ -1,3 +1,3 @@
 {
-  "two": 2
+    "two": 2
 }

geonode/base/api/serializers.py

@@ -28,6 +28,7 @@
 from django.forms.models import model_to_dict
 from django.contrib.auth import get_user_model
 from django.db.models.query import QuerySet
+from geonode.assets.utils import get_default_asset
 from geonode.people import Roles
 from django.http import QueryDict
 from deprecated import deprecated
@@ -62,7 +63,8 @@
 from geonode.geoapps.models import GeoApp
 from geonode.groups.models import GroupCategory, GroupProfile
 from geonode.base.api.fields import ComplexDynamicRelationField
-from geonode.layers.utils import get_dataset_download_handlers, get_default_dataset_download_handler
+from geonode.layers.utils import get_download_handlers, get_default_dataset_download_handler
+from geonode.assets.handlers import asset_handler_registry
 from geonode.utils import build_absolute_uri
 from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes
 from geonode.resource.models import ExecutionRequest
@@ -298,15 +300,24 @@
         except Exception as e:
             logger.exception(e)
             raise e
+
+        asset = get_default_asset(_instance)
+        if asset is not None:
+            asset_url = asset_handler_registry.get_handler(asset).create_download_url(asset)
+
         if _instance.resource_type in ["map"] + get_geoapp_subtypes():
             return []
         elif _instance.resource_type in ["document"]:
-            return [
+            payload = [
                 {
                     "url": _instance.download_url,
                     "ajax_safe": _instance.download_is_ajax_safe,
-                }
+                },
             ]
+            if asset:
+                payload.append({"url": asset_url, "ajax_safe": False, "default": False})
+            return payload
+
         elif _instance.resource_type in ["dataset"]:
             download_urls = []
             # lets get only the default one first to set it
@@ -315,11 +326,14 @@
             if obj.download_url:
                 download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": True})
             # then let's prepare the payload with everything
-            handler_list = get_dataset_download_handlers()
-            for handler in handler_list:
+            for handler in get_download_handlers():
                 obj = handler(self.context.get("request"), _instance.alternate)
                 if obj.download_url:
                     download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False})
+
+            if asset:
+                download_urls.append({"url": asset_url, "ajax_safe": True, "default": False if download_urls else True})
+
             return download_urls
         else:
             return []

geonode/base/api/tests.py

@@ -49,6 +49,8 @@
 from geonode.assets.utils import create_asset_and_link
 from geonode.maps.models import Map, MapLayer
 from geonode.tests.base import GeoNodeBaseTestSupport
+from geonode.assets.utils import get_default_asset
+from geonode.assets.handlers import asset_handler_registry
 
 from geonode.base import enumerations
 from geonode.base.api.serializers import ResourceBaseSerializer
@@ -2288,8 +2290,8 @@
         # clean
         try:
             resource.delete()
         except Exception as e:
             logger.warning(f"Can't delete test resource {resource}", exc_info=e)
 
     def test_resource_service_copy_with_perms_dataset(self):
         files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp")
@@ -2477,7 +2479,12 @@
         Ensure we can access the Resource Base list.
         """
         doc = Document.objects.first()
-        expected_payload = [{"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}]
+        asset = get_default_asset(doc)
+        handler = asset_handler_registry.get_handler(asset)
+        expected_payload = [
+            {"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe},
+            {"ajax_safe": False, "default": False, "url": handler.create_download_url(asset)},
+        ]
         # From resource base API
         json = self._get_for_object(doc, "base-resources-detail")
         download_url = json.get("resource").get("download_urls")

geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py

@@ -18,11 +18,11 @@
 def migrate_files(apps, schema_editor):
 
     def get_ext(filename):
         try:
             return os.path.splitext(filename)[1][1:]
         except Exception as e:
             logger.warning(f"Could not find extension for Resource '{res_hm.title}, file '{filename}': {e}")
             return None
 
     ResourceBase_hm = apps.get_model('base', 'ResourceBase')
     Dataset_hm = apps.get_model('layers', 'Dataset')
@@ -32,44 +32,44 @@
         # looping on available resources with files to generate the LocalAssets
         for res_hm in ResourceBase_hm.objects.exclude(Q(files__isnull=True) | Q(files__exact=[])).iterator():
             # resolving the real owner instance, since resource.owner is an historical model and cant be used directly
             owner = get_user_model().objects.get(pk=res_hm.owner.id)
             # logger.warning(f"Creating ASSET for {resource.id} -- owner:{type(resource.owner)} --> {resource.owner}")
 
             files = res_hm.files
             # creating the local asset object
             asset = LocalAsset(
                 title="Files",
                 description="Original uploaded files",
                 owner=owner,
                 location=files
             )
             asset.save()
 
             ### creating the association between asset and Link
 
             # no existing "uploaded" links exist, so create them right away
             # otherwise we create the link with the assigned asset
             if dataset_hm := Dataset_hm.objects.filter(pk=res_hm.id).first():
-                url = build_absolute_uri(reverse("assets-download", args=(asset.pk,)))
+                url = build_absolute_uri(reverse("assets-link", args=(asset.pk,)))
             elif doc_hm := Document_hm.objects.filter(pk=res_hm.id).first():
                 url = build_absolute_uri(reverse("assets-link", args=(asset.pk,)))
             else:
                 raise TypeError(f'ResourceBase {res_hm.id}::"{res_hm.title} has unhandled type"')
 
             if len(files) == 1:
                 ext = get_ext(files[0])
             else:
                 ext = None
                 for file in files:
                     for filetype in settings.SUPPORTED_DATASET_FILE_TYPES:
                         file_ext = get_ext(file)
                         if file_ext in filetype["ext"]:
                             ext = filetype["id"]
                             break
                     if ext:
                         break
 
             Link.objects.create(
                 resource_id=res_hm.id,
                 asset=asset,
                 link_type="uploaded",

geonode/layers/utils.py

@@ -507,7 +507,7 @@ def get_uuid_handler():
 dataset_download_handler_list = []
 
 
-def get_dataset_download_handlers():
+def get_download_handlers():
     if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None):
         dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0]))
 

geonode/settings.py

@@ -303,7 +303,7 @@
 # Absolute path to the directory that hold assets files
 # This dir should not be made publicly accessible by nginx, since its content may be private
 # Using a sibling of MEDIA_ROOT as default
-ASSETS_ROOT = os.getenv("ASSETS_ROOT", os.path.join(os.path.dirname(MEDIA_ROOT.rstrip("/")), "assets"))
+ASSETS_ROOT = os.getenv("ASSETS_ROOT", os.path.join(os.path.dirname(MEDIA_ROOT.rstrip("/")), "assets_data"))
 
 # Cache Bustin Settings: enable WhiteNoise compression and caching support
 # ref: http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support