feat: eslasticsearch catalog

feat: add back embargoed projects when querying public projects

feat: elasticsearch catalog

feat: add cron command to index catalog & cron job

feat: add geonames_id constraint to Location

fix: call to huma_num with no id during elasticsearch build

feat: divide object dating field in two - era  & period

fix mypy issues

fix(elasticsearch): from query param not passed correctly

feat: catalog data request

chore: move user related test factories to euphro_auth

fix: permission view on data request run inline

feat(es): add is_data_embargoed to RunDoc

feat(data-request): prevent request data from embargoed runs

feat: data request events

feat; add lru cache to eros fetch call

feat: fetch data from eros when building catalog

.env.example

@@ -1,3 +1,4 @@
+CORS_ALLOWED_ORIGINS=http://localhost:8001,http://localhost:8002
 DJANGO_SETTINGS_MODULE=euphrosyne.settings
 DB_USER=
 DB_NAME=euphrosyne
@@ -11,6 +12,9 @@ EMAIL_PORT=1025
 EMAIL_HOST_USER=
 EMAIL_HOST_PASSWORD=
 EMAIL_USE_TLS=false
+ELASTICSEARCH_HOST=http://localhost:9200
+ELASTICSEARCH_USERNAME=
+ELASTICSEARCH_PASSWORD=
 EROS_HTTP_TOKEN=
 EUPHROSYNE_TOOLS_API_URL=http://localhost:8001
 DEFAULT_FROM_EMAIL=alexandre.hajjar@beta.gouv.fr

README.md

@@ -28,10 +28,14 @@ Le contenu du fichier peut-être copié dans un nouveau fichier `.env` pour para
 
 | Nom de la variable       | Description                                                                                                                                                                                                               |
 | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| CORS_ALLOWED_ORIGINS     | Valeur du header Access-Control-Allow-Origin pour les réponses sur les endpoints de l'api REST                                                                                                                            |
 | DJANGO_SETTINGS_MODULE   | Chemin Python vers le module settings Django.                                                                                                                                                                             |
 | DB\_\*                   | Variables relatives à la configuration de la base de données.                                                                                                                                                             |
 | DJANGO_SECRET_KEY        | [Clé secrète](https://docs.djangoproject.com/en/4.1/ref/settings/#std-setting-SECRET_KEY) utilisée par Django pour la signature cryptographique. Est également utilisé pour signer les tokens JWT (Euphrosyne Tools API). |
 | DJANGO_DEBUG             | Optionnel. Mode debug de Django.                                                                                                                                                                                          |
+| ELASTICSEARCH_HOST       | Hôte de l'instance Elasticsearch (catalogue de données)                                                                                                                                                                   |
+| ELASTICSEARCH_USERNAME   | Credentials pour l'instance Elasticsearch (catalogue de données)                                                                                                                                                          |
+| ELASTICSEARCH_PASSWORD   | Credentials pour l'instance Elasticsearch (catalogue de données)                                                                                                                                                          |
 | EMAIL_HOST               | Configuration du service d'e-mail.                                                                                                                                                                                        |
 | EMAIL_PORT               | "                                                                                                                                                                                                                         |
 | EMAIL_HOST_USER          | "                                                                                                                                                                                                                         |

cron.json

@@ -8,6 +8,10 @@
       "command": "0 */6 * * * python manage.py check_project_data_availability",
       "size": "S"
     },
+    {
+      "command": "5 */6 * * * python manage.py index_elasticsearch_catalog",
+      "size": "S"
+    },
     {
       "command": "0 0 * * * python manage.py run_checks",
       "size": "S"

data_request/admin.py

@@ -0,0 +1,182 @@
+from django.contrib import admin, messages
+from django.db.models import Model, QuerySet
+from django.http import HttpRequest
+from django.http.response import HttpResponse
+from django.utils import timezone
+from django.utils.safestring import mark_safe
+from django.utils.translation import gettext
+from django.utils.translation import gettext_lazy as _
+
+from euphro_tools.exceptions import EuphroToolsException
+from lab.admin.mixins import LabAdminAllowedMixin
+from lab.permissions import is_lab_admin
+
+from .data_links import send_links
+from .models import DataAccessEvent, DataRequest
+
+
+class ReadonlyInlineMixin:
+
+    def has_change_permission(
+        self,
+        request: HttpRequest,
+        obj: Model | None = None,  # pylint: disable=unused-argument
+    ) -> bool:
+        return False
+
+    def has_delete_permission(
+        self,
+        request: HttpRequest,
+        obj: Model | None = None,  # pylint: disable=unused-argument
+    ) -> bool:
+        return is_lab_admin(request.user)
+
+    def has_add_permission(
+        self,
+        request: HttpRequest,
+        obj: Model | None = None,  # pylint: disable=unused-argument
+    ) -> bool:
+        return False
+
+    def has_view_permission(
+        self,
+        request: HttpRequest,
+        obj: Model | None = None,  # pylint: disable=unused-argument
+    ) -> bool:
+        return is_lab_admin(request.user)
+
+
+@admin.action(description=_("Accept request(s) (send download links)"))
+def action_send_links(
+    # pylint: disable=unused-argument
+    modeladmin: "DataRequestAdmin",
+    request: HttpRequest,
+    queryset: QuerySet[DataRequest],
+):
+    for data_request in queryset:
+        try:
+            send_links(data_request)
+        except EuphroToolsException as error:
+            modeladmin.message_user(
+                request,
+                _("Error sending links to %(email)s for %(data_request)s: %(error)s")
+                % {
+                    "data_request": data_request,
+                    "error": error,
+                    "email": data_request.user_email,
+                },
+                level=messages.ERROR,
+            )
+            continue
+        data_request.sent_at = timezone.now()
+        if not data_request.request_viewed:
+            data_request.request_viewed = True
+        data_request.save()
+
+
+class BeenSeenListFilter(admin.SimpleListFilter):
+    # Human-readable title which will be displayed in the
+    # right admin sidebar just above the filter options.
+    title = _("has been sent")
+
+    # Parameter for the filter that will be used in the URL query.
+    parameter_name = "been_sent"
+
+    def lookups(self, request, model_admin):
+        return [
+            ("1", _("Yes")),
+            ("0", _("No")),
+        ]
+
+    def queryset(self, request: HttpRequest, queryset: QuerySet[DataRequest]):
+        if not self.value():
+            return queryset
+        return queryset.filter(sent_at__isnull=self.value() == "0")
+
+
+class DataAccessEventInline(ReadonlyInlineMixin, admin.TabularInline):
+    model = DataAccessEvent
+    extra = 0
+
+    fields = ("path", "access_time")
+    readonly_fields = ("path", "access_time")
+
+
+class RunInline(ReadonlyInlineMixin, admin.TabularInline):
+    model = DataRequest.runs.through
+    verbose_name = "Run"
+    verbose_name_plural = "Runs"
+    extra = 0
+
+    fields = ("run",)
+
+
+@admin.register(DataRequest)
+class DataRequestAdmin(LabAdminAllowedMixin, admin.ModelAdmin):
+    actions = [action_send_links]
+    list_filter = [BeenSeenListFilter]
+
+    list_display = (
+        "created",
+        "sent_at",
+        "user_email",
+        "user_first_name",
+        "user_last_name",
+        "display_viewed",
+    )
+
+    fields = (
+        "created",
+        "sent_at",
+        "user_email",
+        "user_first_name",
+        "user_last_name",
+        "user_institution",
+        "description",
+    )
+    readonly_fields = (
+        "created",
+        "user_email",
+        "user_first_name",
+        "user_last_name",
+        "user_institution",
+        "description",
+    )
+
+    inlines = [RunInline, DataAccessEventInline]
+
+    def has_change_permission(self, request: HttpRequest, obj: Model | None = None):
+        return False
+
+    def change_view(
+        self,
+        request: HttpRequest,
+        object_id: str,
+        form_url: str = "",
+        extra_context: dict[str, bool] | None = None,
+    ) -> HttpResponse:
+        obj = self.get_object(request, object_id)
+        if obj and not obj.request_viewed:
+            obj.request_viewed = True
+            obj.save()
+        response = super().change_view(request, object_id, form_url, extra_context)
+        return response
+
+    def changelist_view(
+        self, request: HttpRequest, extra_context: dict[str, str] | None = None
+    ):
+        extra_context = extra_context or {}
+        extra_context["title"] = gettext("Data requests")
+        return super().changelist_view(request, extra_context)
+
+    @admin.display(description=_("Is sent"), boolean=True)
+    def is_sent(self, obj: "DataRequest") -> bool:
+        return obj.sent_at is not None
+
+    @admin.display(description="")
+    def display_viewed(self, obj: "DataRequest") -> str:
+        if obj.request_viewed:
+            return ""
+        return mark_safe(
+            f'<p class="fr-badge fr-badge--new fr-badge--sm">{_("New")}</p>'
+        )

data_request/api_urls.py

@@ -0,0 +1,16 @@
+from django.urls import path
+
+from . import api_views
+
+urlpatterns = (
+    path(
+        "",
+        api_views.DataRequestCreateAPIView.as_view(),
+        name="create",
+    ),
+    path(
+        "access-event",
+        api_views.DataAccessEventCreateAPIView.as_view(),
+        name="create-access-event",
+    ),
+)

data_request/api_views.py

@@ -0,0 +1,52 @@
+from rest_framework import generics, serializers
+
+from data_request.emails import send_data_request_created_email
+from euphro_auth.jwt.authentication import EuphrosyneAdminJWTAuthentication
+from lab.runs.models import Run
+
+from .models import DataAccessEvent, DataRequest
+
+
+class DataRequestSerializer(serializers.ModelSerializer):
+    runs = serializers.PrimaryKeyRelatedField(
+        many=True,
+        queryset=Run.objects.only_not_embargoed(),
+        allow_empty=False,
+    )
+
+    class Meta:
+        model = DataRequest
+        fields = [
+            "user_email",
+            "user_first_name",
+            "user_last_name",
+            "user_institution",
+            "description",
+            "runs",
+        ]
+
+
+class DataRequestCreateAPIView(generics.CreateAPIView):
+    queryset = DataRequest.objects.all()
+    serializer_class = DataRequestSerializer
+
+    def perform_create(self, serializer: DataRequestSerializer):
+        super().perform_create(serializer)
+        send_data_request_created_email(serializer.instance.user_email)
+
+
+class DataAccessEventSerializer(serializers.ModelSerializer):
+    data_request = serializers.PrimaryKeyRelatedField(
+        queryset=DataRequest.objects.all(),
+        allow_empty=False,
+    )
+
+    class Meta:
+        model = DataAccessEvent
+        fields = ["data_request", "path"]
+
+
+class DataAccessEventCreateAPIView(generics.CreateAPIView):
+    queryset = DataAccessEvent.objects.all()
+    serializer_class = DataAccessEventSerializer
+    authentication_classes = [EuphrosyneAdminJWTAuthentication]

data_request/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class DataRequestConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "data_request"

data_request/data_links.py

@@ -0,0 +1,44 @@
+import datetime
+import typing
+
+from euphro_tools.download_urls import (
+    DataType,
+    fetch_token_for_run_data,
+    generate_download_url,
+)
+
+from .emails import LinkDict, send_data_email
+from .models import DataRequest
+
+NUM_DAYS_VALID = 7
+
+
+def send_links(data_request: DataRequest):
+    links: list[LinkDict] = []
+    expiration = datetime.datetime.now() + datetime.timedelta(days=NUM_DAYS_VALID)
+    for run in data_request.runs.all():
+        for data_type in typing.get_args(DataType):
+            project_name = run.project.name
+            token = fetch_token_for_run_data(
+                run.project.slug,
+                run.label,
+                data_type,
+                expiration=expiration,
+                data_request_id=str(data_request.id),
+            )
+            links.append(
+                {
+                    "name": f"{run.label} ({project_name})",
+                    "url": generate_download_url(
+                        data_type=data_type,
+                        project_slug=run.project.slug,
+                        run_label=run.label,
+                        token=token,
+                    ),
+                    "data_type": data_type,
+                }
+            )
+    send_data_email(
+        context={"links": links, "expiration_date": expiration},
+        email=data_request.user_email,
+    )