+
+
+
+
+ {% trans "Mandatory" %}
+
+
+ {% trans "Mandatory" %}
+
+
+ {% trans "Optional" %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
![]()
+
+
+
+ {% block resourcebase_title %}
+
+
+ {{ resourcebase_form.title }}
+ {% endblock %}
+
+
+
+ {{ resourcebase_form.abstract }}
+
+
+
+
+
+
+ {{ resourcebase_form.date_type }}
+
+
+
+
+ {{ resourcebase_form.date }}
+
+ {% block resourcebase_category %}
+
+
+
+
+ {% endblock resourcebase_category %}
+
+
+
+
+
+
+ {{ resourcebase_form.keywords }}
+
+ {% if THESAURI_FILTERS %}
+
+ {{tkeywords_form.as_p}}
+
+ {% endif %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {% block resourcebase_attributes %}
+
+
+
+
+ {{ resourcebase_form.language }}
+
+
+
+
+ {{ resourcebase_form.license }}
+
+
+
+ {{ resourcebase_form.attribution }}
+
+
+ {% endblock resourcebase_attributes %}
+
+
+
+ {{ resourcebase_form.regions }}
+
+
+
+
+ {{ resourcebase_form.data_quality_statement }}
+
+
+
+
+
+
+ {{ resourcebase_form.restriction_code_type }}
+
+
+
+
+ {{ resourcebase_form.constraints_other }}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
{% trans "Other, Optional, Metadata" %}
+
+
+
+ {{ resourcebase_form.edition }}
+
+
+
+ {{ resourcebase_form.doi }}
+
+
+
+
+ {{ resourcebase_form.purpose }}
+
+
+
+
+ {{ resourcebase_form.supplemental_information }}
+
+
+
+ {% block resourcebase_temporal_extent_start %}
+
+
+
+
+ {{ resourcebase_form.temporal_extent_start }}
+
+
+ {% endblock resourcebase_temporal_extent_start %}
+ {% block resourcebase_temporal_extent_end %}
+
+
+
+
+ {{ resourcebase_form.temporal_extent_end }}
+
+
+ {% endblock resourcebase_temporal_extent_end %}
+ {% block maintenance_block %}
+
+
+
+
+ {{ resourcebase_form.maintenance_frequency }}
+
+
+
+
+ {{ resourcebase_form.spatial_representation_type }}
+
+ {% block resourcebase_extra_metadata %}
+
+
+ {{ resourcebase_form.extra_metadata }}
+
+ {% endblock resourcebase_extra_metadata %}
+ {% block resourcebase_linked_resources %}
+
+
+ {{ resourcebase_form.linked_resources }}
+
+ {% endblock resourcebase_linked_resources %}
+
+
+ {% endblock maintenance_block %}
+
+
+ {% block resourcebase_poc %}
+
+
{% trans "Responsible Parties" %}
+
+
+ {{ resourcebase_form.poc }}
+
+
+ {% endblock %}
+
+
{% trans "Responsible and Permissions" %}
+ {% block resourcebase_owner %}
+
+
+
+ {{ resourcebase_form.owner }}
+
+ {% endblock resourcebase_owner %}
+
+
+
{% trans "toggle more Contact Roles" %}
+ {% block resourcebase_more_contact_roles %}
+
+
+ {% endblock resourcebase_more_contact_roles %}
+
+
+
+
+
+ {% block extra_metadata_content %}
+ {% endblock %}
+
+
+
+
+
+
+
+
+
{% trans "Publishing" %}
+
+
+
+ {{ resourcebase_form.metadata_uploaded_preserve }}
+
+
+
+ {{ resourcebase_form.is_approved }}
+
+
+
+ {{ resourcebase_form.is_published }}
+
+ {% if user.is_superuser %}
+
+
+ {{ resourcebase_form.featured }}
+
+ {% endif %}
+
+
+ {{ resourcebase_form.advertised }}
+
+
+
+
+
+
+
+
+
+
+
+
+
{% trans "Other Settings" %}
+
+
+
+ {{ resourcebase_form.id }}
+
+
+
+
+
+
+
+
+
+{% endblock %}
diff --git a/geonode/base/tests/data/img.gif b/geonode/base/tests/data/img.gif
new file mode 100644
index 00000000000..56959b6411a
Binary files /dev/null and b/geonode/base/tests/data/img.gif differ
diff --git a/geonode/base/views.py b/geonode/base/views.py
index e51c342fedf..64adc5bf202 100644
--- a/geonode/base/views.py
+++ b/geonode/base/views.py
@@ -18,6 +18,9 @@
#########################################################################
import json
import logging
+import ast
+import warnings
+import traceback
from dal import views, autocomplete
from user_messages.models import Message
@@ -28,6 +31,8 @@
from django.shortcuts import render
from django.http import HttpResponse
from django.views.generic import FormView
+from django.core.exceptions import ObjectDoesNotExist
+from django.views.decorators.clickjacking import xframe_options_sameorigin
from django.http import HttpResponseRedirect
from django.contrib.auth import get_user_model
from django.contrib import messages
@@ -55,6 +60,21 @@
from geonode.base.forms import BatchEditForm, OwnerRightsRequestForm
from geonode.base.models import Region, ResourceBase, HierarchicalKeyword, ThesaurusKeyword, ThesaurusKeywordLabel
+from geonode.base.enumerations import SOURCE_TYPE_LOCAL
+
+from geonode.client.hooks import hookset
+from geonode.people.forms import ProfileForm
+from geonode.monitoring.models import EventType
+from geonode.base.auth import get_or_create_token
+from geonode.security.views import _perms_info_json
+from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager
+from geonode.decorators import check_keyword_write_perms
+
+from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm
+from geonode.base.models import Thesaurus, TopicCategory
+
+from .forms import ResourceBaseForm
+
logger = logging.getLogger(__name__)
@@ -479,3 +499,347 @@ def resource_clone(request):
status_code = 400
return HttpResponse(json.dumps(out), content_type="application/json", status=status_code)
+
+
+logger = logging.getLogger("geonode.base.metadata")
+
+_PERMISSION_MSG_GENERIC = _("You do not have permissions for this resource.")
+_PERMISSION_MSG_METADATA = _("You are not allowed to modify this resource's metadata.")
+_PERMISSION_MSG_VIEW = _("You are not allowed to view this resource.")
+
+
+def _resolve_resourcebase(request, id, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs):
+ """
+ Resolve the resourcebase by the provided typename and check the optional permission.
+ """
+
+ return resolve_object(request, ResourceBase, {"pk": id}, permission=permission, permission_msg=msg, **kwargs)
+
+
+@xframe_options_sameorigin
+def resourcebase_embed(request, resourcebaseid, template="base/base_edit.html"):
+ """
+ The view that returns the app composer opened to
+ the app with the given app ID.
+ """
+ try:
+ resourcebase_obj = _resolve_resourcebase(
+ request, resourcebaseid, "base.view_resourcebase", _PERMISSION_MSG_VIEW
+ )
+ except PermissionDenied:
+ return HttpResponse(_("Not allowed"), status=403)
+ except Exception:
+ raise Http404(_("Not found"))
+ if not resourcebase_obj:
+ raise Http404(_("Not found"))
+
+ # Call this first in order to be sure "perms_list" is correct
+ permissions_json = _perms_info_json(resourcebase_obj)
+
+ perms_list = list(
+ resourcebase_obj.get_self_resource()
+ .get_user_perms(request.user)
+ .union(resourcebase_obj.get_user_perms(request.user))
+ )
+
+ group = None
+ if resourcebase_obj.group:
+ try:
+ group = GroupProfile.objects.get(slug=resourcebase_obj.group.name)
+ except GroupProfile.DoesNotExist:
+ group = None
+
+ r = resourcebase_obj
+ if request.method in ("POST", "PATCH", "PUT"):
+ r = resource_manager.update(resourcebase_obj.uuid, instance=resourcebase_obj, notify=True)
+
+ resource_manager.set_permissions(
+ resourcebase_obj.uuid, instance=resourcebase_obj, permissions=ast.literal_eval(permissions_json)
+ )
+
+ resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False)
+
+ access_token = None
+ if request and request.user:
+ access_token = get_or_create_token(request.user)
+ if access_token and not access_token.is_expired():
+ access_token = access_token.token
+ else:
+ access_token = None
+
+ _config = json.dumps(r.blob)
+ _ctx = {
+ "appId": resourcebaseid,
+ "appType": resourcebase_obj.resource_type,
+ "config": _config,
+ "user": request.user,
+ "access_token": access_token,
+ "resource": resourcebase_obj,
+ "group": group,
+ "perms_list": perms_list,
+ "permissions_json": permissions_json,
+ "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"),
+ }
+
+ return render(request, template, context=_ctx)
+
+
+def resourcebase_metadata_detail(
+ request, resourcebaseid, template="base/base_metadata_detail.html", custom_metadata=None
+):
+ try:
+ resourcebase_obj = _resolve_resourcebase(request, resourcebaseid, "view_resourcebase", _PERMISSION_MSG_METADATA)
+ except PermissionDenied:
+ return HttpResponse(_("Not allowed"), status=403)
+ except Exception:
+ raise Http404(_("Not found"))
+ if not resourcebase_obj:
+ raise Http404(_("Not found"))
+
+ group = None
+ if resourcebase_obj.group:
+ try:
+ group = GroupProfile.objects.get(slug=resourcebase_obj.group.name)
+ except ObjectDoesNotExist:
+ group = None
+ site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL
+ register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj)
+
+ return render(
+ request,
+ template,
+ context={
+ "resource": resourcebase_obj,
+ "group": group,
+ "SITEURL": site_url,
+ "custom_metadata": custom_metadata,
+ },
+ )
+
+
+@login_required
+@check_keyword_write_perms
+def resourcebase_metadata(
+ request,
+ resourcebaseid,
+ template="base/base_metadata.html",
+ ajax=True,
+ panel_template="base/base_panels.html",
+ custom_metadata=None,
+):
+ resourcebase_obj = None
+ try:
+ resourcebase_obj = _resolve_resourcebase(
+ request, resourcebaseid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA
+ )
+ except PermissionDenied:
+ return HttpResponse(_("Not allowed"), status=403)
+ except Exception:
+ raise Http404(_("Not found"))
+ if not resourcebase_obj:
+ raise Http404(_("Not found"))
+
+ # Add metadata_author or poc if missing
+ resourcebase_obj.add_missing_metadata_author_or_poc()
+ resource_type = resourcebase_obj.resource_type
+ topic_category = resourcebase_obj.category
+ subtype = resourcebase_obj.subtype
+ current_keywords = [keyword.name for keyword in resourcebase_obj.keywords.all()]
+
+ topic_thesaurus = resourcebase_obj.tkeywords.all()
+
+ if request.method == "POST":
+ resourcebase_form = ResourceBaseForm(
+ request.POST, instance=resourcebase_obj, prefix="resource", user=request.user
+ )
+ category_form = CategoryForm(
+ request.POST,
+ prefix="category_choice_field",
+ initial=(
+ int(request.POST["category_choice_field"])
+ if "category_choice_field" in request.POST and request.POST["category_choice_field"]
+ else None
+ ),
+ )
+
+ if hasattr(settings, "THESAURUS"):
+ tkeywords_form = TKeywordForm(request.POST)
+ else:
+ tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords")
+
+ else:
+ resourcebase_form = ResourceBaseForm(instance=resourcebase_obj, prefix="resource", user=request.user)
+ resourcebase_form.disable_keywords_widget_for_non_superuser(request.user)
+ category_form = CategoryForm(
+ prefix="category_choice_field", initial=topic_category.id if topic_category else None
+ )
+
+ # Create THESAURUS widgets
+ lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en"
+ if hasattr(settings, "THESAURUS") and settings.THESAURUS:
+ warnings.warn(
+ "The settings for Thesaurus has been moved to Model, \
+ this feature will be removed in next releases",
+ DeprecationWarning,
+ )
+ dataset_tkeywords = resourcebase_obj.tkeywords.all()
+ tkeywords_list = ""
+ if dataset_tkeywords and len(dataset_tkeywords) > 0:
+ tkeywords_ids = dataset_tkeywords.values_list("id", flat=True)
+ if hasattr(settings, "THESAURUS") and settings.THESAURUS:
+ el = settings.THESAURUS
+ thesaurus_name = el["name"]
+ try:
+ t = Thesaurus.objects.get(identifier=thesaurus_name)
+ for tk in t.thesaurus.filter(pk__in=tkeywords_ids):
+ tkl = tk.keyword.filter(lang=lang)
+ if len(tkl) > 0:
+ tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True)))
+ tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids
+ except Exception:
+ tb = traceback.format_exc()
+ logger.error(tb)
+ tkeywords_form = TKeywordForm(instance=resourcebase_obj)
+ else:
+ tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords")
+ # set initial values for thesaurus form
+ for tid in tkeywords_form.fields:
+ values = []
+ values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id]
+ tkeywords_form.fields[tid].initial = values
+
+ if (
+ request.method == "POST"
+ and resourcebase_form.is_valid()
+ and category_form.is_valid()
+ and tkeywords_form.is_valid()
+ ):
+ new_keywords = current_keywords if request.keyword_readonly else resourcebase_form.cleaned_data.pop("keywords")
+ new_regions = resourcebase_form.cleaned_data.pop("regions")
+
+ new_category = None
+ if (
+ category_form
+ and "category_choice_field" in category_form.cleaned_data
+ and category_form.cleaned_data["category_choice_field"]
+ ):
+ new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"]))
+ resourcebase_form.cleaned_data.pop("ptype")
+
+ resourcebase_obj = resourcebase_form.instance
+ # update contact roles
+ resourcebase_obj.set_contact_roles_from_metadata_edit(resourcebase_form)
+
+ vals = dict(category=new_category, subtype=subtype)
+
+ resourcebase_form.cleaned_data.pop("metadata")
+ extra_metadata = resourcebase_form.cleaned_data.pop("extra_metadata")
+
+ resourcebase_form.save_linked_resources()
+ resourcebase_form.cleaned_data.pop("linked_resources")
+
+ vals.update({"resource_type": resource_type, "sourcetype": SOURCE_TYPE_LOCAL})
+
+ register_event(request, EventType.EVENT_CHANGE_METADATA, resourcebase_obj)
+ if not ajax:
+ return HttpResponseRedirect(hookset.resourcebase_detail_url(resourcebase_obj))
+
+ message = resourcebase_obj.id
+
+ try:
+ # Keywords from THESAURUS management
+ # Rewritten to work with updated autocomplete
+ if not tkeywords_form.is_valid():
+ return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400))
+
+ thesaurus_setting = getattr(settings, "THESAURUS", None)
+ if thesaurus_setting:
+ tkeywords_data = tkeywords_form.cleaned_data["tkeywords"]
+ tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"])
+ resourcebase_obj.tkeywords.set(tkeywords_data)
+ elif Thesaurus.objects.all().exists():
+ fields = tkeywords_form.cleaned_data
+ resourcebase_obj.tkeywords.set(tkeywords_form.cleanx(fields))
+
+ except Exception:
+ tb = traceback.format_exc()
+ logger.error(tb)
+
+ if "group" in resourcebase_form.changed_data:
+ vals["group"] = resourcebase_form.cleaned_data.get("group")
+ if any([x in resourcebase_form.changed_data for x in ["is_approved", "is_published"]]):
+ vals["is_approved"] = resourcebase_form.cleaned_data.get("is_approved", resourcebase_obj.is_approved)
+ vals["is_published"] = resourcebase_form.cleaned_data.get("is_published", resourcebase_obj.is_published)
+ else:
+ vals.pop("is_approved", None)
+ vals.pop("is_published", None)
+
+ resource_manager.update(
+ resourcebase_obj.uuid,
+ instance=resourcebase_obj,
+ keywords=new_keywords,
+ regions=new_regions,
+ notify=True,
+ vals=vals,
+ extra_metadata=json.loads(extra_metadata),
+ )
+
+ resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False)
+
+ return HttpResponse(json.dumps({"message": message}))
+ elif request.method == "POST" and (
+ not resourcebase_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid()
+ ):
+ errors_list = {
+ **resourcebase_form.errors.as_data(),
+ **category_form.errors.as_data(),
+ **tkeywords_form.errors.as_data(),
+ }
+ logger.error(f"resourcebase Metadata form is not valid: {errors_list}")
+ out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]}
+ return HttpResponse(json.dumps(out), content_type="application/json", status=400)
+ # - POST Request Ends here -
+
+ # define contact role forms
+ contact_role_forms_context = {}
+ for role in resourcebase_obj.get_multivalue_role_property_names():
+ resourcebase_form.fields[role].initial = [p.username for p in resourcebase_obj.__getattribute__(role)]
+ role_form = ProfileForm(prefix=role)
+ role_form.hidden = True
+ contact_role_forms_context[f"{role}_form"] = role_form
+
+ metadata_author_groups = get_user_visible_groups(request.user)
+
+ if not AdvancedSecurityWorkflowManager.is_allowed_to_publish(request.user, resourcebase_obj):
+ resourcebase_form.fields["is_published"].widget.attrs.update({"disabled": "true"})
+ if not AdvancedSecurityWorkflowManager.is_allowed_to_approve(request.user, resourcebase_obj):
+ resourcebase_form.fields["is_approved"].widget.attrs.update({"disabled": "true"})
+
+ register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj)
+ return render(
+ request,
+ template,
+ context={
+ "resource": resourcebase_obj,
+ "resourcebase": resourcebase_obj,
+ "panel_template": panel_template,
+ "custom_metadata": custom_metadata,
+ "resourcebase_form": resourcebase_form,
+ "category_form": category_form,
+ "tkeywords_form": tkeywords_form,
+ "metadata_author_groups": metadata_author_groups,
+ "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False),
+ "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False),
+ "UI_MANDATORY_FIELDS": list(
+ set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", []))
+ | set(getattr(settings, "UI_REQUIRED_FIELDS", []))
+ ),
+ **contact_role_forms_context,
+ "UI_ROLES_IN_TOGGLE_VIEW": resourcebase_obj.get_ui_toggled_role_property_names(),
+ },
+ )
+
+
+@login_required
+def resourcebase_metadata_advanced(request, resourcebaseid):
+ return resourcebase_metadata(request, resourcebaseid, template="base/base_metadata_advanced.html")
diff --git a/geonode/client/hooksets.py b/geonode/client/hooksets.py
index 84fcc5ec4b3..4745a4601fd 100644
--- a/geonode/client/hooksets.py
+++ b/geonode/client/hooksets.py
@@ -119,6 +119,9 @@ def geoapp_list_url(self):
def geoapp_detail_url(self, geoapp):
return NotImplemented
+ def resourcebase_embed_template(self, context=None):
+ return None
+
# Documents
def document_list_url(self):
return NotImplemented
@@ -139,3 +142,6 @@ def metadata_update_redirect(self, url, request=None):
if "metadata_uri" in url:
return url.replace("/metadata_uri", "")
return url.replace("/metadata", "")
+
+ def get_absolute_url(self, context=None):
+ return None
diff --git a/geonode/client/templatetags/client_lib_tags.py b/geonode/client/templatetags/client_lib_tags.py
index 3ce659a32bf..2ec9837b06f 100644
--- a/geonode/client/templatetags/client_lib_tags.py
+++ b/geonode/client/templatetags/client_lib_tags.py
@@ -187,6 +187,11 @@ def render(self, context):
elif self.tag_name == "get_geoapp_download":
t = context.template.engine.get_template(hookset.geoapp_download_template(context=context))
+ # 3DTILES
+
+ if self.tag_name == "get_resourcebase_embed":
+ t = context.template.engine.get_template(hookset.resourcebase_embed_template(context=context))
+
if t:
return t.render(context)
else:
@@ -225,3 +230,5 @@ def do_get_client_library_template(parser, token):
register.tag("get_geoapp_update", do_get_client_library_template)
register.tag("get_geoapp_embed", do_get_client_library_template)
register.tag("get_geoapp_download", do_get_client_library_template)
+
+register.tag("get_resourcebase_embed", do_get_client_library_template)
diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py
index d8fe1ca2395..d68ca3a09b8 100644
--- a/geonode/documents/api/views.py
+++ b/geonode/documents/api/views.py
@@ -28,6 +28,7 @@
from oauth2_provider.contrib.rest_framework import OAuth2Authentication
from geonode import settings
+from geonode.assets.utils import create_asset_and_link
from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter
from geonode.base.api.mixins import AdvertisedListMixin
from geonode.base.api.pagination import GeoNodeApiPagination
@@ -47,6 +48,7 @@
import logging
+
logger = logging.getLogger(__name__)
@@ -119,16 +121,20 @@ def perform_create(self, serializer):
"extension": extension,
"resource_type": "document",
}
- if file:
- manager = StorageManager(remote_files={"base_file": file})
- manager.clone_remote_files()
- payload["files"] = [manager.get_retrieved_paths().get("base_file")]
if doc_url:
payload["doc_url"] = doc_url
payload["sourcetype"] = enumerations.SOURCE_TYPE_REMOTE
resource = serializer.save(**payload)
+ if file:
+ manager = StorageManager(remote_files={"base_file": file})
+ manager.clone_remote_files()
+ create_asset_and_link(
+ resource, self.request.user, [manager.get_retrieved_paths().get("base_file")], clone_files=True
+ )
+ manager.delete_retrieved_paths(force=True)
+
resource.set_missing_info()
resourcebase_post_save(resource.get_real_instance())
resource_manager.set_permissions(None, instance=resource, permissions=None, created=True)
@@ -136,6 +142,7 @@ def perform_create(self, serializer):
resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False)
return resource
except Exception as e:
+ logger.error(f"Error creating document {serializer.validated_data}", exc_info=e)
if manager:
manager.delete_retrieved_paths()
raise e
diff --git a/geonode/documents/models.py b/geonode/documents/models.py
index 25cc5c2b86d..cdd069929a5 100644
--- a/geonode/documents/models.py
+++ b/geonode/documents/models.py
@@ -26,6 +26,7 @@
from django.utils.functional import classproperty
from django.utils.translation import gettext_lazy as _
+from geonode.assets.models import Asset
from geonode.client.hooks import hookset
from geonode.base.models import ResourceBase
from geonode.groups.conf import settings as groups_settings
@@ -76,6 +77,11 @@ def compact_permission_labels(cls):
"owner": _("Owner"),
}
+ @property
+ def files(self):
+ asset = Asset.objects.filter(link__resource=self).first()
+ return asset.location if asset else []
+
@property
def name(self):
if not self.title:
diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py
index e0ed9617354..316bdafdfe8 100644
--- a/geonode/documents/tasks.py
+++ b/geonode/documents/tasks.py
@@ -26,6 +26,8 @@
from geonode.celery_app import app
from geonode.storage.manager import StorageManager
+from geonode.assets.handlers import asset_handler_registry
+from geonode.assets.utils import get_default_asset
from ..base.models import ResourceBase
from .models import Document
@@ -90,7 +92,7 @@ def create_document_thumbnail(self, object_id):
"""
logger.debug(f"Generating thumbnail for document #{object_id}.")
- storage_manager = StorageManager()
+ default_storage_manager = StorageManager()
try:
document = Document.objects.get(id=object_id)
@@ -104,15 +106,24 @@ def create_document_thumbnail(self, object_id):
centering = (0.5, 0.5)
doc_path = None
- if document.files:
- doc_path = storage_manager.path(document.files[0])
+
+ # get asset of the resource
+ asset = get_default_asset(document)
+ if not asset and not document.doc_url:
+ raise Exception("Document has neither an associated Asset nor a link, cannot generate thumbnail")
+
+ if asset:
+ handler = asset_handler_registry.get_handler(asset)
+ asset_storage_manager = handler.get_storage_manager(asset)
+ doc_path = asset_storage_manager.path(asset.location[0])
elif document.doc_url:
doc_path = document.doc_url
remove_tmp_file = True
+ asset_storage_manager = default_storage_manager
if document.is_image:
try:
- image_file = storage_manager.open(doc_path)
+ image_file = asset_storage_manager.open(doc_path)
except Exception as e:
logger.debug(f"Could not generate thumbnail from remote document {document.doc_url}: {e}")
@@ -129,12 +140,12 @@ def create_document_thumbnail(self, object_id):
if image_file is not None:
image_file.close()
if remove_tmp_file:
- storage_manager.delete(doc_path)
+ default_storage_manager.delete(doc_path)
elif doc_renderer.supports(doc_path):
# in case it's a remote document we want to retrieve it first
if document.doc_url:
- doc_path = storage_manager.open(doc_path).name
+ doc_path = default_storage_manager.open(doc_path).name
remove_tmp_file = True
try:
thumbnail_content = doc_renderer.render(doc_path)
@@ -145,7 +156,7 @@ def create_document_thumbnail(self, object_id):
print(e)
finally:
if remove_tmp_file:
- storage_manager.delete(doc_path)
+ default_storage_manager.delete(doc_path)
if not thumbnail_content:
logger.warning(f"Thumbnail for document #{object_id} empty.")
ResourceBase.objects.filter(id=document.id).update(thumbnail_url=None)
diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py
index e8adaa83c6e..69a82b78903 100644
--- a/geonode/documents/tests.py
+++ b/geonode/documents/tests.py
@@ -42,6 +42,8 @@
from guardian.shortcuts import get_anonymous_user
+from geonode.assets.utils import create_asset_and_link
+from geonode.base.forms import LinkedResourceForm
from geonode.maps.models import Map
from geonode.layers.models import Dataset
from geonode.compat import ensure_string
@@ -58,7 +60,9 @@
from geonode.upload.api.exceptions import FileUploadLimitException
from .forms import DocumentCreateForm
-from ..base.forms import LinkedResourceForm
+
+
+TEST_GIF = os.path.join(os.path.dirname(__file__), "tests/data/img.gif")
class DocumentsTest(GeoNodeBaseTestSupport):
@@ -113,10 +117,10 @@ def test_document_mimetypes_rendering(self):
def test_create_document_with_no_rel(self, thumb):
"""Tests the creation of a document with no relations"""
thumb.return_value = True
- f = [f"{settings.MEDIA_ROOT}/img.gif"]
superuser = get_user_model().objects.get(pk=2)
- c = Document.objects.create(files=f, owner=superuser, title="theimg")
+ c = Document.objects.create(owner=superuser, title="theimg")
+ _, _ = create_asset_and_link(c, superuser, [TEST_GIF])
c.set_default_permissions()
self.assertEqual(Document.objects.get(pk=c.id).title, "theimg")
@@ -412,11 +416,11 @@ def test_ajax_document_permissions(self, create_thumb):
"""Verify that the ajax_document_permissions view is behaving as expected"""
create_thumb.return_value = True
# Setup some document names to work with
- f = [f"{settings.MEDIA_ROOT}/img.gif"]
-
superuser = get_user_model().objects.get(pk=2)
document = resource_manager.create(
- None, resource_type=Document, defaults=dict(files=f, owner=superuser, title="theimg", is_approved=True)
+ None,
+ resource_type=Document,
+ defaults=dict(files=[TEST_GIF], owner=superuser, title="theimg", is_approved=True),
)
document_id = document.id
invalid_document_id = 20
@@ -630,10 +634,10 @@ def setUp(self):
def test_create_document_with_links(self):
"""Tests the creation of document links."""
- f = [f"{settings.MEDIA_ROOT}/img.gif"]
superuser = get_user_model().objects.get(pk=2)
- d = Document.objects.create(files=f, owner=superuser, title="theimg")
+ d = Document.objects.create(owner=superuser, title="theimg")
+ _, _ = create_asset_and_link(d, superuser, [TEST_GIF])
self.assertEqual(Document.objects.get(pk=d.id).title, "theimg")
@@ -679,11 +683,10 @@ def setUp(self):
self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True)
self.not_admin.set_password("very-secret")
self.not_admin.save()
- self.files = [f"{settings.MEDIA_ROOT}/img.gif"]
self.test_doc = resource_manager.create(
None,
resource_type=Document,
- defaults=dict(files=self.files, owner=self.not_admin, title="test", is_approved=True),
+ defaults=dict(files=[TEST_GIF], owner=self.not_admin, title="test", is_approved=True),
)
self.perm_spec = {"users": {"AnonymousUser": []}}
self.doc_link_url = reverse("document_link", args=(self.test_doc.pk,))
@@ -808,7 +811,7 @@ def test_document_link_with_permissions(self):
# Access resource with user logged-in
self.client.login(username=self.not_admin.username, password="very-secret")
response = self.client.get(self.doc_link_url)
- self.assertEqual(response.status_code, 404)
+ self.assertEqual(response.status_code, 200)
# test document link with external url
doc = resource_manager.create(
None,
diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py
index 63facbbaf09..8f5ec4ad619 100644
--- a/geonode/documents/utils.py
+++ b/geonode/documents/utils.py
@@ -23,6 +23,9 @@
# Standard Modules
import os
import logging
+
+from geonode.assets.handlers import asset_handler_registry
+from geonode.assets.utils import get_default_asset
from geonode.storage.manager import storage_manager
# Django functionality
@@ -31,7 +34,6 @@
from django.template import loader
from django.utils.translation import gettext_lazy as _
from django.utils.text import slugify
-from django_downloadview.response import DownloadResponse
# Geonode functionality
from geonode.documents.models import Document
@@ -78,10 +80,6 @@ def get_download_response(request, docid, attachment=False):
register_event(request, EventType.EVENT_DOWNLOAD, document)
filename = slugify(os.path.splitext(os.path.basename(document.title))[0])
- if document.files and storage_manager.exists(document.files[0]):
- return DownloadResponse(
- storage_manager.open(document.files[0]).file,
- basename=f"{filename}.{document.extension}",
- attachment=attachment,
- )
- return HttpResponse("File is not available", status=404)
+ asset = get_default_asset(document)
+ asset_handler = asset_handler_registry.get_handler(asset)
+ return asset_handler.get_download_handler(asset).create_response(asset, attachment, basename=filename)
diff --git a/geonode/documents/views.py b/geonode/documents/views.py
index 545fd715647..411f80a4bd0 100644
--- a/geonode/documents/views.py
+++ b/geonode/documents/views.py
@@ -33,8 +33,10 @@
from django.views.generic.edit import CreateView, UpdateView
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
-from geonode.base.api.exceptions import geonode_exception_handler
+from geonode.assets.handlers import asset_handler_registry
+from geonode.assets.utils import get_default_asset
+from geonode.base.api.exceptions import geonode_exception_handler
from geonode.client.hooks import hookset
from geonode.utils import mkdtemp, resolve_object
from geonode.base.views import batch_modify
@@ -169,12 +171,21 @@ def form_valid(self, form):
owner=self.request.user,
doc_url=doc_form.pop("doc_url", None),
title=doc_form.pop("title", file.name),
+ description=doc_form.pop("abstract", None),
extension=doc_form.pop("extension", None),
+ link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES
+ data_title=doc_form.pop("title", file.name),
+ data_type=doc_form.pop("extension", None),
files=[storage_path],
),
)
- if tempdir != os.path.dirname(storage_path):
- shutil.rmtree(tempdir, ignore_errors=True)
+
+ # Removing the temp file
+ # TODO: creating a file and then cloning it as an Asset may be slow: we may want to
+ # create the file directly in the asset dir or to move it
+ logger.info(f"Removing document temp dir {tempdir}")
+ shutil.rmtree(tempdir, ignore_errors=True)
+
else:
self.object = resource_manager.create(
None,
@@ -278,11 +289,17 @@ def form_valid(self, form):
if file:
tempdir = mkdtemp()
dirname = os.path.basename(tempdir)
- filepath = storage_manager.save(f"{dirname}/{file.name}", file)
+ filepath = storage_manager.save(os.path.join(dirname, file.name), file)
storage_path = storage_manager.path(filepath)
self.object = resource_manager.update(
- self.object.uuid, instance=self.object, vals=dict(owner=self.request.user, files=[storage_path])
+ self.object.uuid, instance=self.object, vals=dict(owner=self.request.user)
)
+
+ # replace data in existing asset
+ asset = get_default_asset(self.object, link_type="uploaded")
+ if asset:
+ asset_handler_registry.get_handler(asset).replace_data(asset, [storage_path])
+
if tempdir != os.path.dirname(storage_path):
shutil.rmtree(tempdir, ignore_errors=True)
diff --git a/geonode/geoserver/manager.py b/geonode/geoserver/manager.py
index a88b641b71f..e3cc4a69378 100644
--- a/geonode/geoserver/manager.py
+++ b/geonode/geoserver/manager.py
@@ -290,7 +290,6 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non
_to_update = {
"name": _name,
"title": instance.title or _gs_import_session_info.dataset_name,
- "files": kwargs.get("files", None),
"workspace": _gs_import_session_info.workspace,
"alternate": _alternate,
"typename": _alternate,
diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py
index 85672984b00..7d2b9ac2155 100644
--- a/geonode/geoserver/tests/test_manager.py
+++ b/geonode/geoserver/tests/test_manager.py
@@ -19,6 +19,7 @@
import os
import base64
import shutil
+from django.test import override_settings
import gisdata
import requests
@@ -54,8 +55,14 @@ def tearDown(self) -> None:
return super().tearDown()
@on_ogc_backend(geoserver.BACKEND_PACKAGE)
+ @override_settings(ASYNC_SIGNALS=False, FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777, FILE_UPLOAD_PERMISSIONS=0o7777)
def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self):
layer = Dataset.objects.get(name=self.sut.name)
+ gs_layer = self.cat.get_layer("san_andres_y_providencia_water")
+ if gs_layer is None:
+ _gs_import_session_info = self.geoserver_manager._execute_resource_import(
+ layer, list(self.files_as_dict.values()), self.user, action_type="create"
+ )
_gs_import_session_info = self.geoserver_manager._execute_resource_import(
layer, list(self.files_as_dict.values()), self.user, action_type="append"
)
diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py
index 53b4bf7af76..b7102116c9e 100644
--- a/geonode/layers/utils.py
+++ b/geonode/layers/utils.py
@@ -507,7 +507,7 @@ def get_uuid_handler():
dataset_download_handler_list = []
-def get_dataset_download_handlers():
+def get_download_handlers():
if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None):
dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0]))
diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py
index 6ad3febe547..5b7beafc906 100644
--- a/geonode/maps/api/views.py
+++ b/geonode/maps/api/views.py
@@ -130,6 +130,7 @@ def perform_create(self, serializer):
create_action_perfomed=True,
additional_data=post_creation_data,
)
+
# Handle thumbnail generation
resource_manager.set_thumbnail(instance.uuid, instance=instance, overwrite=False)
diff --git a/geonode/proxy/templatetags/proxy_lib_tags.py b/geonode/proxy/templatetags/proxy_lib_tags.py
index 1135d12bd86..5993d7ef67e 100644
--- a/geonode/proxy/templatetags/proxy_lib_tags.py
+++ b/geonode/proxy/templatetags/proxy_lib_tags.py
@@ -17,6 +17,7 @@
#
#########################################################################
+from geonode.assets.utils import get_default_asset
from geonode.base.models import ResourceBase
import traceback
@@ -52,7 +53,10 @@ def original_link_available(context, resourceid, url):
dataset_files = []
if isinstance(instance, ResourceBase):
try:
- for file in instance.files:
+ asset_obj = get_default_asset(instance)
+ # Copy all Dataset related files into a temporary folder
+ files = asset_obj.location if asset_obj else []
+ for file in files:
dataset_files.append(file)
if not storage_manager.exists(file):
return False
diff --git a/geonode/proxy/tests.py b/geonode/proxy/tests.py
index 2f094488ee0..20a0b091b29 100644
--- a/geonode/proxy/tests.py
+++ b/geonode/proxy/tests.py
@@ -30,6 +30,7 @@
from urllib.parse import urljoin
from django.conf import settings
+from geonode.assets.utils import create_asset_and_link
from geonode.proxy.templatetags.proxy_lib_tags import original_link_available
from django.test.client import RequestFactory
from django.core.files.uploadedfile import SimpleUploadedFile
@@ -308,12 +309,15 @@ def test_download_url_with_existing_files(self, fopen, fexists):
fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc")
dataset = Dataset.objects.all().first()
- dataset.files = [
+ dataset_files = [
"/tmpe1exb9e9/foo_file.dbf",
"/tmpe1exb9e9/foo_file.prj",
"/tmpe1exb9e9/foo_file.shp",
"/tmpe1exb9e9/foo_file.shx",
]
+ asset, link = create_asset_and_link(
+ dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False
+ )
dataset.save()
@@ -331,6 +335,9 @@ def test_download_url_with_existing_files(self, fopen, fexists):
self.assertEqual("application/zip", response.headers.get("Content-Type"))
self.assertEqual('attachment; filename="CA.zip"', response.headers.get("Content-Disposition"))
+ link.delete()
+ asset.delete()
+
@patch("geonode.storage.manager.storage_manager.exists")
@patch("geonode.storage.manager.storage_manager.open")
@on_ogc_backend(geoserver.BACKEND_PACKAGE)
@@ -339,12 +346,15 @@ def test_download_files(self, fopen, fexists):
fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc")
dataset = Dataset.objects.all().first()
- dataset.files = [
+ dataset_files = [
"/tmpe1exb9e9/foo_file.dbf",
"/tmpe1exb9e9/foo_file.prj",
"/tmpe1exb9e9/foo_file.shp",
"/tmpe1exb9e9/foo_file.shx",
]
+ asset, link = create_asset_and_link(
+ dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False
+ )
dataset.save()
@@ -368,6 +378,9 @@ def test_download_files(self, fopen, fexists):
self.assertIn(".shx", "".join(zip_files))
self.assertIn(".prj", "".join(zip_files))
+ link.delete()
+ asset.delete()
+
class OWSApiTestCase(GeoNodeBaseTestSupport):
def setUp(self):
@@ -420,16 +433,23 @@ def test_should_return_true_if_files_are_available(self, fexists):
assert upload
- self.resource.files = [
+ dataset_files = [
"/tmpe1exb9e9/foo_file.dbf",
"/tmpe1exb9e9/foo_file.prj",
"/tmpe1exb9e9/foo_file.shp",
"/tmpe1exb9e9/foo_file.shx",
]
+ asset, link = create_asset_and_link(
+ self.resource, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False
+ )
+
self.resource.save()
self.resource.refresh_from_db()
actual = original_link_available(self.context, self.resource.resourcebase_ptr_id, self.url)
self.assertTrue(actual)
+
+ link.delete()
+ asset.delete()
diff --git a/geonode/proxy/views.py b/geonode/proxy/views.py
index b7bca6ee5b9..20fe0158ca5 100644
--- a/geonode/proxy/views.py
+++ b/geonode/proxy/views.py
@@ -55,6 +55,7 @@
from geonode.base import register_event
from geonode.base.auth import get_auth_user, get_token_from_auth_header
from geonode.geoserver.helpers import ogc_server_settings
+from geonode.assets.utils import get_default_asset
from .utils import proxy_urls_registry
@@ -245,8 +246,9 @@ def download(request, resourceid, sender=Dataset):
dataset_files = []
file_list = [] # Store file info to be returned
try:
- files = instance.resourcebase_ptr.files
+ asset_obj = get_default_asset(instance)
# Copy all Dataset related files into a temporary folder
+ files = asset_obj.location if asset_obj else []
for file_path in files:
if storage_manager.exists(file_path):
dataset_files.append(file_path)
diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py
index d4c94a1bc66..eefab451b4a 100644
--- a/geonode/resource/manager.py
+++ b/geonode/resource/manager.py
@@ -21,6 +21,7 @@
import copy
import typing
import logging
+import itertools
from uuid import uuid1, uuid4
from abc import ABCMeta, abstractmethod
@@ -38,6 +39,8 @@
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist, ValidationError, FieldDoesNotExist
+
+from geonode.base.models import ResourceBase, LinkedResource
from geonode.thumbs.thumbnails import _generate_thumbnail_name
from geonode.documents.tasks import create_document_thumbnail
from geonode.security.permissions import PermSpecCompact, DATA_STYLABLE_RESOURCES_SUBTYPES
@@ -45,9 +48,9 @@
from . import settings as rm_settings
from .utils import update_resource, resourcebase_post_save
+from geonode.assets.utils import create_asset_and_link_dict, rollback_asset_and_link, copy_assets_and_links, create_link
from ..base import enumerations
-from ..base.models import ResourceBase, LinkedResource
from ..security.utils import AdvancedSecurityWorkflowManager
from ..layers.metadata import parse_metadata
from ..documents.models import Document
@@ -313,20 +316,35 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de
if resource_type.objects.filter(uuid=uuid).exists():
return resource_type.objects.filter(uuid=uuid).get()
uuid = uuid or str(uuid4())
- _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=defaults)
+ resource_dict = { # TODO: cleanup params and dicts
+ k: v
+ for k, v in defaults.items()
+ if k not in ("data_title", "data_type", "description", "files", "link_type", "extension", "asset")
+ }
+ _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=resource_dict)
if _resource and _created:
_resource.set_processing_state(enumerations.STATE_RUNNING)
try:
+ # if files exist: create an Asset out of them and link it to the Resource
+ asset, link = (None, None) # safe init in case of exception
+ if defaults.get("files", None):
+ logger.debug(f"Found files when creating resource {_resource}: {defaults['files']}")
+ asset, link = create_asset_and_link_dict(_resource, defaults, clone_files=True)
+ elif defaults.get("asset", None):
+ logger.debug(f"Found asset when creating resource {_resource}: {defaults['asset']}")
+ link = create_link(_resource, **defaults)
+
with transaction.atomic():
_resource.set_missing_info()
_resource = self._concrete_resource_manager.create(
- uuid, resource_type=resource_type, defaults=defaults
+ uuid, resource_type=resource_type, defaults=resource_dict
)
_resource.save()
resourcebase_post_save(_resource.get_real_instance())
_resource.set_processing_state(enumerations.STATE_PROCESSED)
except Exception as e:
logger.exception(e)
+ rollback_asset_and_link(asset, link) # we are not removing the Asset passed in defaults
self.delete(_resource.uuid, instance=_resource)
raise e
return _resource
@@ -440,19 +458,19 @@ def ingest(
) -> ResourceBase:
instance = None
to_update = defaults.copy()
- if "files" in to_update:
- to_update.pop("files")
+ to_update_with_files = {**to_update, **{"files": files}}
try:
with transaction.atomic():
if resource_type == Document:
if "name" in to_update:
to_update.pop("name")
- if files:
- to_update["files"] = storage_manager.copy_files_list(files)
- instance = self.create(uuid, resource_type=Document, defaults=to_update)
+ instance = self.create(uuid, resource_type=Document, defaults=to_update_with_files)
elif resource_type == Dataset:
if files:
- instance = self.create(uuid, resource_type=Dataset, defaults=to_update)
+ instance = self.create(uuid, resource_type=Dataset, defaults=to_update_with_files)
+ else:
+ logger.warning(f"Will not create a Dataset without any file. Values: {defaults}")
+
if instance:
instance = self._concrete_resource_manager.ingest(
storage_manager.copy_files_list(files),
@@ -523,11 +541,15 @@ def copy(
_maplayer.pk = _maplayer.id = None
_maplayer.map = _resource.get_real_instance()
_maplayer.save()
+
+ assets_and_links = copy_assets_and_links(instance, target=_resource)
+ # we're just merging all the files together: it won't work once we have multiple assets per resource
+ # TODO: get the files from the proper Asset, or make the _concrete_resource_manager.copy use assets
to_update = {}
- try:
- to_update = storage_manager.copy(_resource).copy()
- except Exception as e:
- logger.exception(e)
+
+ files = list(itertools.chain.from_iterable([asset.location for asset, _ in assets_and_links]))
+ if files:
+ to_update = {"files": files}
_resource = self._concrete_resource_manager.copy(instance, uuid=_resource.uuid, defaults=to_update)
diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py
index 4b25308afbe..a1850861673 100644
--- a/geonode/resource/tests.py
+++ b/geonode/resource/tests.py
@@ -148,7 +148,15 @@ def _copy_assert_resource(res, title):
# copy with documents
res = self.rm.ingest(
- dt_files, resource_type=Document, defaults={"title": "relief_san_andres", "owner": self.user}
+ dt_files,
+ resource_type=Document,
+ defaults={
+ "title": "relief_san_andres",
+ "owner": self.user,
+ "extension": "tif",
+ "data_title": "relief_san_andres",
+ "data_type": "tif",
+ },
)
self.assertTrue(isinstance(res, Document))
_copy_assert_resource(res, "Testing Document 2")
@@ -157,7 +165,12 @@ def _copy_assert_resource(res, title):
res = self.rm.ingest(
dt_files,
resource_type=Dataset,
- defaults={"owner": self.user, "title": "Testing Dataset", "files": dt_files},
+ defaults={
+ "owner": self.user,
+ "title": "Testing Dataset",
+ "data_title": "relief_san_andres",
+ "data_type": "tif",
+ },
)
self.assertTrue(isinstance(res, Dataset))
_copy_assert_resource(res, "Testing Dataset 2")
diff --git a/geonode/resource/utils.py b/geonode/resource/utils.py
index f0e8e2201e5..78e215173e8 100644
--- a/geonode/resource/utils.py
+++ b/geonode/resource/utils.py
@@ -29,9 +29,11 @@
from django.utils import timezone
from django.core.exceptions import FieldDoesNotExist
from django.utils.translation import gettext_lazy as _
-from geonode.utils import OGC_Servers_Handler
from django.utils.module_loading import import_string
+from geonode.assets.utils import get_default_asset
+from geonode.utils import OGC_Servers_Handler
+
from ..base import enumerations
from ..base.models import (
ExtraMetadata,
@@ -241,10 +243,13 @@ def update_resource(
]
to_update.update(defaults)
+ resource_dict = { # TODO: cleanup params and dicts
+ k: v for k, v in to_update.items() if k not in ("data_title", "data_type", "description", "files", "link_type")
+ }
try:
- instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**to_update)
+ instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**resource_dict)
except Exception as e:
- logger.error(f"{e} - {to_update}")
+ logger.error(f"{e} - {resource_dict}")
raise
# Check for "remote services" availability
@@ -322,9 +327,9 @@ def get_alternate_name(instance):
def document_post_save(instance, *args, **kwargs):
instance.csw_type = "document"
-
- if instance.files:
- _, extension = os.path.splitext(os.path.basename(instance.files[0]))
+ asset = get_default_asset(instance)
+ if asset:
+ _, extension = os.path.splitext(os.path.basename(asset.location[0]))
instance.extension = extension[1:]
doc_type_map = DOCUMENT_TYPE_MAP
doc_type_map.update(getattr(settings, "DOCUMENT_TYPE_MAP", {}))
@@ -344,7 +349,7 @@ def document_post_save(instance, *args, **kwargs):
mime = mime_type_map.get(ext, "text/plain")
url = None
- if instance.id and instance.files:
+ if instance.id and asset:
name = "Hosted Document"
site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL
url = f"{site_url}{reverse('document_download', args=(instance.id,))}"
@@ -455,8 +460,10 @@ def resourcebase_post_save(instance, *args, **kwargs):
if hasattr(instance, "abstract") and not getattr(instance, "abstract", None):
instance.abstract = _("No abstract provided")
if hasattr(instance, "title") and not getattr(instance, "title", None) or getattr(instance, "title", "") == "":
- if isinstance(instance, Document) and instance.files:
- instance.title = os.path.basename(instance.files[0])
+ asset = get_default_asset(instance)
+ files = asset.location if asset else []
+ if isinstance(instance, Document) and files:
+ instance.title = os.path.basename(files[0])
if hasattr(instance, "name") and getattr(instance, "name", None):
instance.title = instance.name
if (
diff --git a/geonode/security/tests.py b/geonode/security/tests.py
index 5ae00733cf4..ce9b40a507f 100644
--- a/geonode/security/tests.py
+++ b/geonode/security/tests.py
@@ -21,9 +21,11 @@
import base64
import logging
import uuid
+import os
import requests
import importlib
import mock
+import gisdata
from requests.auth import HTTPBasicAuth
from tastypie.test import ResourceTestCaseMixin
@@ -40,7 +42,9 @@
from guardian.shortcuts import assign_perm, get_anonymous_user
from geonode import geoserver
-from geonode.geoserver.helpers import geofence, gf_utils
+from geonode.geoserver.helpers import geofence, gf_utils, gs_catalog
+from geonode.geoserver.manager import GeoServerResourceManager
+from geonode.layers.utils import get_files
from geonode.maps.models import Map
from geonode.layers.models import Dataset
from geonode.documents.models import Document
@@ -742,8 +746,19 @@ def test_perm_specs_synchronization(self):
@on_ogc_backend(geoserver.BACKEND_PACKAGE)
def test_dataset_permissions(self):
# Test permissions on a layer
+ files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_poi.shp")
+ files_as_dict, self.tmpdir = get_files(files)
+
bobby = get_user_model().objects.get(username="bobby")
- layer = create_single_dataset("san_andres_y_providencia_poi")
+ layer = create_single_dataset(
+ "san_andres_y_providencia_poi",
+ {
+ "owner": self.user,
+ "title": "Testing Dataset",
+ "data_title": "relief_san_andres",
+ "data_type": "tif",
+ },
+ )
layer = resource_manager.update(
layer.uuid, instance=layer, notify=False, vals=dict(owner=bobby, workspace=settings.DEFAULT_WORKSPACE)
)
@@ -774,6 +789,15 @@ def test_dataset_permissions(self):
perm_spec = {"users": {"AnonymousUser": []}, "groups": []}
layer.set_permissions(perm_spec)
+ gs_layer = gs_catalog.get_layer("3Asan_andres_y_providencia_poi")
+ if gs_layer is None:
+ GeoServerResourceManager()._execute_resource_import(
+ layer,
+ list(files_as_dict.values()),
+ get_user_model().objects.get(username="admin"),
+ action_type="create",
+ )
+
url = (
f"{settings.GEOSERVER_LOCATION}ows?"
"LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES="
@@ -786,7 +810,8 @@ def test_dataset_permissions(self):
# test view_resourcebase permission on anonymous user
response = requests.get(url)
- self.assertTrue(response.status_code, 404)
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(b"Could not find layer" in response.content)
self.assertEqual(response.headers.get("Content-Type"), "application/vnd.ogc.se_xml;charset=UTF-8")
# test WMS with authenticated user that has access to the Dataset
@@ -796,7 +821,7 @@ def test_dataset_permissions(self):
username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"]
),
)
- self.assertTrue(response.status_code, 200)
+ self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get("Content-Type"), "image/png")
# test WMS with authenticated user that has no view_resourcebase:
diff --git a/geonode/settings.py b/geonode/settings.py
index 84333dc7ef2..098dd86ba79 100644
--- a/geonode/settings.py
+++ b/geonode/settings.py
@@ -300,6 +300,11 @@
# Example: "/home/media/media.lawrence.com/apps/"
STATIC_ROOT = os.getenv("STATIC_ROOT", os.path.join(PROJECT_ROOT, "static_root"))
+# Absolute path to the directory that hold assets files
+# This dir should not be made publicly accessible by nginx, since its content may be private
+# Using a sibling of MEDIA_ROOT as default
+ASSETS_ROOT = os.getenv("ASSETS_ROOT", os.path.join(os.path.dirname(MEDIA_ROOT.rstrip("/")), "assets_data"))
+
# Cache Bustin Settings: enable WhiteNoise compression and caching support
# ref: http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support
CACHE_BUSTING_STATIC_ENABLED = ast.literal_eval(os.environ.get("CACHE_BUSTING_STATIC_ENABLED", "False"))
@@ -701,7 +706,7 @@
},
"loggers": {
"django": {
- "level": "ERROR",
+ "level": "WARN",
},
"geonode": {
"level": "WARN",
@@ -2340,6 +2345,7 @@ def get_geonode_catalogue_service():
'importer.handlers.geotiff.handler.GeoTiffFileHandler',\
'importer.handlers.xml.handler.XMLFileHandler',\
'importer.handlers.sld.handler.SLDFileHandler',\
+ 'importer.handlers.tiles3d.handler.Tiles3DFileHandler',\
]",
)
)
@@ -2365,3 +2371,10 @@ def get_geonode_catalogue_service():
AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS = ast.literal_eval(
os.getenv("AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS", "True")
)
+
+DEFAULT_ASSET_HANDLER = "geonode.assets.local.LocalAssetHandler"
+ASSET_HANDLERS = [
+ DEFAULT_ASSET_HANDLER,
+]
+INSTALLED_APPS += ("geonode.assets",)
+GEONODE_APPS += ("geonode.assets",)
diff --git a/geonode/storage/data_retriever.py b/geonode/storage/data_retriever.py
index bc23997a8a2..f7178937228 100644
--- a/geonode/storage/data_retriever.py
+++ b/geonode/storage/data_retriever.py
@@ -151,10 +151,12 @@ def __init__(self, files, tranfer_at_creation=False):
if tranfer_at_creation:
self.transfer_remote_files()
- def transfer_remote_files(self):
+ def transfer_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True):
from geonode.utils import mkdtemp
- self.temporary_folder = mkdtemp()
+ self.temporary_folder = cloning_directory or settings.MEDIA_ROOT
+ if create_tempdir:
+ self.temporary_folder = mkdtemp(cloning_directory or settings.MEDIA_ROOT, prefix=prefix)
for name, data_item_retriever in self.data_items.items():
file_path = data_item_retriever.transfer_remote_file(self.temporary_folder)
self.file_paths[name] = Path(file_path)
@@ -172,10 +174,12 @@ def transfer_remote_files(self):
os.chmod(self.temporary_folder, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
return self.file_paths
- def get_paths(self, allow_transfer=False):
+ def get_paths(self, allow_transfer=False, cloning_directory=None, prefix=None, create_tempdir=True):
if not self.file_paths:
if allow_transfer:
- self.transfer_remote_files()
+ self.transfer_remote_files(
+ cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir
+ )
else:
raise DataRetrieverExcepion(detail="You can't retrieve paths without clone file first!")
return self.file_paths.copy()
diff --git a/geonode/storage/manager.py b/geonode/storage/manager.py
index 2966fc6924e..4d1cbbb0133 100644
--- a/geonode/storage/manager.py
+++ b/geonode/storage/manager.py
@@ -124,8 +124,8 @@ class StorageManager(StorageManagerInterface):
treat as a file_system file
"""
- def __init__(self, remote_files: Mapping = {}):
- self._concrete_storage_manager = self._get_concrete_manager()
+ def __init__(self, remote_files: Mapping = {}, concrete_storage_manager=None):
+ self._concrete_storage_manager = concrete_storage_manager or self._get_concrete_manager()
self.data_retriever = DataRetriever(remote_files, tranfer_at_creation=False)
def _get_concrete_manager(self):
@@ -174,18 +174,19 @@ def replace(self, resource, files: Union[list, BinaryIO]):
updated_files["files"] = [self.replace_single_file(resource.files[0], files)]
return updated_files
- def copy(self, resource):
- updated_files = {}
- if len(resource.files):
- updated_files["files"] = self.copy_files_list(resource.files)
- return updated_files
+ def copy(self, resource, target=None):
+ raise Exception("This is not the copy you're looking for")
+ # updated_files = {}
+ # if len(resource.files):
+ # updated_files["files"] = self.copy_files_list(resource.files)
+ # return updated_files
- def copy_files_list(self, files: List[str]):
+ def copy_files_list(self, files: List[str], dir=settings.MEDIA_ROOT, dir_prefix=None, dir_suffix=None):
from geonode.utils import mkdtemp
out = []
random_suffix = f"{uuid1().hex[:8]}"
- new_path = mkdtemp()
+ new_path = mkdtemp(dir=dir, prefix=dir_prefix, suffix=dir_suffix)
if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
# value is always set by default as None
@@ -242,11 +243,13 @@ def replace_single_file(self, old_file: str, new_file: BinaryIO, prefix: str = N
def generate_filename(self, filename):
return self._concrete_storage_manager.generate_filename(filename)
- def clone_remote_files(self) -> Mapping:
+ def clone_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True) -> Mapping:
"""
Using the data retriever object clone the remote path into a local temporary storage
"""
- return self.data_retriever.get_paths(allow_transfer=True)
+ return self.data_retriever.get_paths(
+ allow_transfer=True, cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir
+ )
def get_retrieved_paths(self) -> Mapping:
"""
@@ -266,8 +269,8 @@ def delete_retrieved_paths(self, force=False) -> None:
class DefaultStorageManager(StorageManagerInterface):
- def __init__(self):
- self._fsm = FileSystemStorage()
+ def __init__(self, **kwargs):
+ self._fsm = FileSystemStorage(**kwargs)
def _get_concrete_manager(self):
return DefaultStorageManager()
diff --git a/geonode/storage/tests.py b/geonode/storage/tests.py
index 9a96186ab3e..43adc794ffa 100644
--- a/geonode/storage/tests.py
+++ b/geonode/storage/tests.py
@@ -401,22 +401,6 @@ def test_storage_manager_replace_single_file(self, path, strg):
output = self.sut().replace(dataset, new_file)
self.assertListEqual([expected], output["files"])
- @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777)
- @override_settings(FILE_UPLOAD_PERMISSIONS=0o777)
- def test_storage_manager_copy(self):
- """
- Test that the copy works as expected and the permissions are corerct
- """
- dataset = create_single_dataset(name="test_copy")
- dataset.files = [os.path.join(f"{self.project_root}", "tests/data/test_sld.sld")]
- dataset.save()
- output = self.sut().copy(dataset)
-
- self.assertTrue(os.path.exists(output.get("files")[0]))
- self.assertEqual(os.stat(os.path.exists(output.get("files")[0])).st_mode, 8592)
- os.remove(output.get("files")[0])
- self.assertFalse(os.path.exists(output.get("files")[0]))
-
class TestDataRetriever(TestCase):
@classmethod
diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py
index c607799d07d..75a4dee41fc 100644
--- a/geonode/upload/api/tests.py
+++ b/geonode/upload/api/tests.py
@@ -231,6 +231,8 @@ def live_upload_file(self, _file):
return response, response.content
@override_settings(CELERY_TASK_ALWAYS_EAGER=True)
+ @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777)
+ @override_settings(FILE_UPLOAD_PERMISSIONS=0o777)
def test_rest_uploads(self):
"""
Ensure we can access the Local Server Uploads list.
@@ -255,12 +257,21 @@ def test_rest_uploads(self):
self.assertEqual(len(response.data["uploads"]), 0)
logger.debug(response.data)
except Exception:
- if resp.json().get("errors"):
- layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0]
+ json = resp.json()
+ if json.get("errors"):
+ logger.error(f"Error in upload: {json}")
+ try:
+ layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0]
+ except IndexError as e:
+ logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e)
+ # TODO: make sure the _cleanup_layer will use the proper layer name
+ self.skipTest("Error with GeoServer")
finally:
self._cleanup_layer(layer_name)
@override_settings(CELERY_TASK_ALWAYS_EAGER=True)
+ @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777)
+ @override_settings(FILE_UPLOAD_PERMISSIONS=0o777)
def test_rest_uploads_non_interactive(self):
"""
Ensure we can access the Local Server Uploads list.
@@ -276,9 +287,16 @@ def test_rest_uploads_non_interactive(self):
exec_id = data.get("execution_id", None)
_exec = ExecutionRequest.objects.get(exec_id=exec_id)
self.assertEqual(_exec.status, "finished")
- except Exception:
- if resp.json().get("errors"):
- layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0]
+ except Exception as e:
+ json = resp.json()
+ logger.warning(f"Error with GeoServer {json}: {e}", exc_info=e)
+ if json.get("errors"):
+ try:
+ layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0]
+ except IndexError as e:
+ logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e)
+ # TODO: make sure the _cleanup_layer will use the proper layer name
+ self.skipTest("Error with GeoServer")
finally:
self._cleanup_layer(layer_name)
diff --git a/geonode/urls.py b/geonode/urls.py
index ccab35e950f..99b1e5ebaf9 100644
--- a/geonode/urls.py
+++ b/geonode/urls.py
@@ -69,6 +69,7 @@
urlpatterns += [
# ResourceBase views
re_path(r"^base/", include("geonode.base.urls")),
+ re_path(r"^resources/", include("geonode.base.base_urls")),
# Dataset views
re_path(r"^datasets/", include("geonode.layers.urls")),
# Remote Services views
@@ -127,6 +128,7 @@
re_path(r"^api/v2/", include("geonode.management_commands_http.urls")),
re_path(r"^api/v2/api-auth/", include("rest_framework.urls", namespace="geonode_rest_framework")),
re_path(r"^api/v2/", include("geonode.facets.urls")),
+ re_path(r"^api/v2/", include("geonode.assets.urls")),
re_path(r"", include(api.urls)),
]
diff --git a/geonode/utils.py b/geonode/utils.py
index c93b61d7cf7..9268feb51da 100755
--- a/geonode/utils.py
+++ b/geonode/utils.py
@@ -281,13 +281,13 @@ def all(self):
return [self[alias] for alias in self]
-def mkdtemp(dir=settings.MEDIA_ROOT):
+def mkdtemp(dir=settings.MEDIA_ROOT, prefix=None, suffix=None):
if not os.path.exists(dir):
os.makedirs(dir, exist_ok=True)
tempdir = None
while not tempdir:
try:
- tempdir = tempfile.mkdtemp(dir=dir)
+ tempdir = tempfile.mkdtemp(dir=dir, prefix=prefix, suffix=suffix)
if os.path.exists(tempdir) and os.path.isdir(tempdir):
if os.listdir(tempdir):
raise Exception("Directory is not empty")
diff --git a/tasks.py b/tasks.py
index b19fa5f7973..82233ee642d 100755
--- a/tasks.py
+++ b/tasks.py
@@ -341,8 +341,9 @@ def statics(ctx):
try:
static_root = os.environ.get("STATIC_ROOT", "/mnt/volumes/statics/static/")
media_root = os.environ.get("MEDIA_ROOT", "/mnt/volumes/statics/uploaded/")
+ assets_root = os.environ.get("ASSETS_ROOT", "/mnt/volumes/statics/assets/")
- ctx.run(f"mkdir -pv {static_root} {media_root}")
+ ctx.run(f"mkdir -pv {static_root} {media_root} {assets_root}")
ctx.run(
f"python manage.py collectstatic --noinput --settings={_localsettings()}",
pty=True,