From af991ea26e19e81cc21686381607e3050914fb04 Mon Sep 17 00:00:00 2001 From: witold Date: Tue, 25 Jun 2024 11:24:24 +0200 Subject: [PATCH] feat: divide object dating field in two - era & period --- .../dating-open-theso-type-ahead.ts | 31 ++++ .../js/web-components/material-type-ahead.ts | 37 ++-- .../web-components/open-theso-type-ahead.ts | 44 +++++ .../js/web-components/period-type-ahead.ts | 37 ---- lab/api_views/serializers.py | 5 + lab/elasticsearch/catalog.py | 58 +++++-- lab/elasticsearch/documents.py | 13 +- lab/elasticsearch/queries.py | 14 +- lab/elasticsearch/tests/_mock.py | 32 ++-- lab/elasticsearch/tests/test_catalog.py | 44 +++-- lab/elasticsearch/tests/test_client.py | 4 +- ...period_unique_theso_joconde_id_and_more.py | 160 ++++++++++++++++++ lab/objects/assets/js/services.ts | 5 +- lab/objects/c2rmf.py | 6 +- lab/objects/forms.py | 44 ++--- lab/objects/models.py | 40 ++--- .../tests/forms/test_objectgroup_form.py | 41 +++-- lab/objects/tests/test_c2mrf.py | 4 +- lab/objects/tests/test_object_formset.py | 2 +- lab/objects/widgets.py | 33 +++- lab/static/css/admin/objectgroup.css | 2 +- .../js/widgets/dating-autocomplete-widget.js | 13 +- .../widgets/dating_autocomplete_widget.html | 33 +++- lab/tests/factories.py | 12 +- lab/tests/test_opentheso.py | 4 +- lab/thesauri/__init__.py | 0 lab/thesauri/models.py | 40 +++++ lab/{ => thesauri}/opentheso.py | 10 +- pylint.txt | 0 29 files changed, 566 insertions(+), 202 deletions(-) create mode 100644 euphrosyne/assets/js/web-components/dating-open-theso-type-ahead.ts create mode 100644 euphrosyne/assets/js/web-components/open-theso-type-ahead.ts delete mode 100644 euphrosyne/assets/js/web-components/period-type-ahead.ts create mode 100644 lab/migrations/0039_era_remove_period_period_unique_theso_joconde_id_and_more.py create mode 100644 lab/thesauri/__init__.py create mode 100644 lab/thesauri/models.py rename lab/{ => thesauri}/opentheso.py (79%) create mode 100644 pylint.txt diff --git a/euphrosyne/assets/js/web-components/dating-open-theso-type-ahead.ts b/euphrosyne/assets/js/web-components/dating-open-theso-type-ahead.ts new file mode 100644 index 000000000..d358f7756 --- /dev/null +++ b/euphrosyne/assets/js/web-components/dating-open-theso-type-ahead.ts @@ -0,0 +1,31 @@ +import { Result } from "../type-ahead-list.component"; +import { + OpenThesoTypeAhead, + SearchType, + OpenThesoResult, +} from "./open-theso-type-ahead"; + +export class DatingOpenThesoTypeAhead extends OpenThesoTypeAhead { + searchType: SearchType = "fullpathSearch"; + + connectedCallback(): void { + this.thesorusId = this.getAttribute("thesorus-id") || ""; + super.connectedCallback(); + } + + async fetchResults(query: string): Promise { + const data = await this.doFetch(query); + return data.map((item: OpenThesoResult[]) => ({ + label: item.map((i) => i.label).join(" > "), + id: item.slice(-1)[0].id, + })) as Result[]; + } +} + +customElements.define( + "dating-open-theso-type-ahead", + DatingOpenThesoTypeAhead, + { + extends: "div", + }, +); diff --git a/euphrosyne/assets/js/web-components/material-type-ahead.ts b/euphrosyne/assets/js/web-components/material-type-ahead.ts index 6130cf6b8..125b6fce9 100644 --- a/euphrosyne/assets/js/web-components/material-type-ahead.ts +++ b/euphrosyne/assets/js/web-components/material-type-ahead.ts @@ -1,34 +1,21 @@ -import { TypeAheadList, Result } from "../type-ahead-list.component"; - -interface OpenThesoResult { - id: string; - arkId: string; - label: string; -} +import { Result } from "../type-ahead-list.component"; +import { + OpenThesoResult, + OpenThesoTypeAhead, + SearchType, +} from "./open-theso-type-ahead"; // eslint-disable-next-line @typescript-eslint/no-unused-vars -class MaterialTypeAhead extends TypeAheadList { - async fetchResults(query: string): Promise { - const q = encodeURIComponent(query); - const response = await fetch( - `https://opentheso.huma-num.fr/opentheso/openapi/v1/concept/th291/autocomplete/${q}?lang=fr&exactMatch=false`, - ); +class MaterialTypeAhead extends OpenThesoTypeAhead { + thesorusId = "th291"; + searchType: SearchType = "autocomplete"; - if (response && response.status === 404) { - return []; - } - if (!response || !response.ok) { - throw new Error("Failed to fetch results"); - } - - const data = await response.json(); - if (!data || !data.length) { - return []; - } + async fetchResults(query: string): Promise { + const data = await this.doFetch(query); return data.map((item: OpenThesoResult) => ({ label: item.label, id: item.id, - })); + })) as Result[]; } } diff --git a/euphrosyne/assets/js/web-components/open-theso-type-ahead.ts b/euphrosyne/assets/js/web-components/open-theso-type-ahead.ts new file mode 100644 index 000000000..019e00373 --- /dev/null +++ b/euphrosyne/assets/js/web-components/open-theso-type-ahead.ts @@ -0,0 +1,44 @@ +import { TypeAheadList } from "../type-ahead-list.component"; + +export type SearchType = "fullpathSearch" | "autocomplete"; + +export interface OpenThesoResult { + id: string; + arkId: string; + label: string; +} + +export abstract class OpenThesoTypeAhead extends TypeAheadList { + thesorusId?: string; + abstract searchType: SearchType; + + async doFetch(query: string): Promise { + if (!this.thesorusId) { + throw new Error("thesorus-id attribute is required"); + } + const q = encodeURIComponent(query); + + let url; + + if (this.searchType === "fullpathSearch") { + url = `https://opentheso.huma-num.fr/opentheso/openapi/v1/concept/${this.thesorusId}/search/fullpath?q=${q}&lang=fr&exactMatch=false`; + } else { + // autocomplete + url = `https://opentheso.huma-num.fr/opentheso/openapi/v1/concept/${this.thesorusId}/autocomplete/${q}?lang=fr&exactMatch=false`; + } + const response = await fetch(url); + + if (response && response.status === 404) { + return []; + } + if (!response || !response.ok) { + throw new Error("Failed to fetch results"); + } + + const data = await response.json(); + if (!data || !data.length) { + return []; + } + return data; + } +} diff --git a/euphrosyne/assets/js/web-components/period-type-ahead.ts b/euphrosyne/assets/js/web-components/period-type-ahead.ts deleted file mode 100644 index 98a30c2a0..000000000 --- a/euphrosyne/assets/js/web-components/period-type-ahead.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { TypeAheadList, Result } from "../type-ahead-list.component"; - -interface OpenThesoResult { - id: string; - arkId: string; - label: string; -} - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -class PeriodTypeAhead extends TypeAheadList { - async fetchResults(query: string): Promise { - const q = encodeURIComponent(query); - const response = await fetch( - `https://opentheso.huma-num.fr/opentheso/openapi/v1/concept/th289/search/fullpath?q=${q}&lang=fr&exactMatch=false`, - ); - - if (response && response.status === 404) { - return []; - } - if (!response || !response.ok) { - throw new Error("Failed to fetch results"); - } - - const data = await response.json(); - if (!data || !data.length) { - return []; - } - return data.map((item: OpenThesoResult[]) => ({ - label: item.map((i) => i.label).join(" > "), - id: item.slice(-1)[0].id, - })); - } -} - -customElements.define("period-type-ahead", PeriodTypeAhead, { - extends: "div", -}); diff --git a/lab/api_views/serializers.py b/lab/api_views/serializers.py index 5552029f4..f294e79ba 100644 --- a/lab/api_views/serializers.py +++ b/lab/api_views/serializers.py @@ -117,10 +117,15 @@ class Meta: class _RunObjectGroupObjectGroupSerializer(serializers.ModelSerializer): + dating = serializers.SerializerMethodField() + class Meta: model = ObjectGroup fields = ("label", "id", "object_count", "dating", "materials") + def get_dating(self, obj: ObjectGroup): + return obj.dating_era.label if obj.dating_era else "" + class RunObjectGroupSerializer(serializers.ModelSerializer): objectgroup = _RunObjectGroupObjectGroupSerializer() diff --git a/lab/elasticsearch/catalog.py b/lab/elasticsearch/catalog.py index 4eb376011..744f8e932 100644 --- a/lab/elasticsearch/catalog.py +++ b/lab/elasticsearch/catalog.py @@ -5,9 +5,12 @@ from lab.methods.dto import method_model_to_dto from lab.models import ObjectGroup, Project -from lab.opentheso import fetch_epoques_parent_ids_from_id from lab.participations.models import Participation from lab.runs.models import Run +from lab.thesauri.opentheso import ( + fetch_era_parent_ids_from_id, + fetch_period_parent_ids_from_id, +) from .documents import ( CatalogItem, @@ -26,6 +29,15 @@ class LocationDict(TypedDict): lon: float +class DatingDict(TypedDict, total=False): + dating_period_label: str | None + dating_period_theso_huma_num_id: str | None + dating_period_theso_huma_num_parent_ids: list[str] | None + dating_era_label: str | None + dating_era_theso_huma_num_id: str | None + dating_era_theso_huma_num_parent_ids: list[str] | None + + def _create_leader_doc(leader: Participation): doc = LeaderDoc( user_first_name=leader.user.first_name, @@ -55,9 +67,12 @@ def _create_project_page_data( ) for object_group in object_groups: - dating_label: str | None = None - if object_group.dating: - dating_label = object_group.dating.label + dating_era_label: str | None = None + dating_period_label: str | None = None + if object_group.dating_period: + dating_period_label = object_group.dating_period.label + if object_group.dating_era: + dating_era_label = object_group.dating_era.label discovery_place_label: str | None = None if object_group.discovery_place_location: @@ -71,8 +86,9 @@ def _create_project_page_data( "materials": object_group.materials, "discovery_place_label": discovery_place_label, "collection": object_group.collection, - "dating_label": dating_label, "inventory": object_group.inventory, + "dating_period_label": dating_period_label, + "dating_era_label": dating_era_label, } ), objects=list( @@ -162,17 +178,25 @@ def build_object_group_catalog_document( locations = [location_geopoint] # Dating - dating_label: str | None = None - dating_theso_huma_num_id: str | None = None - dating_theso_huma_num_parent_ids: list[str] | None = [] - if object_group.dating: - dating_label = object_group.dating.label - dating_theso_huma_num_id = object_group.dating.theso_joconde_id - dating_theso_huma_num_parent_ids = None - if dating_theso_huma_num_id: - dating_theso_huma_num_parent_ids = fetch_epoques_parent_ids_from_id( - object_group.dating.theso_joconde_id + dating_dict = {} + for field_name in ["dating_period", "dating_era"]: + fetch_parent_ids_fn = ( + fetch_era_parent_ids_from_id + if field_name == "dating_era" + else fetch_period_parent_ids_from_id + ) + if getattr(object_group, field_name): + theso_huma_num_parent_ids = fetch_parent_ids_fn( + getattr(object_group, field_name).concept_id ) + dating_dict = { + **dating_dict, + f"{field_name}_label": getattr(object_group, field_name).label, + f"{field_name}_theso_huma_num_id": getattr( + object_group, field_name + ).concept_id, + f"{field_name}_theso_huma_num_parent_ids": theso_huma_num_parent_ids, + } _id = f"object-{object_group.id}" catalog_item = CatalogItem( meta={"id": _id}, @@ -191,9 +215,7 @@ def build_object_group_catalog_document( discovery_place_label=location_label, discovery_place_point=location_geopoint, discovery_place_points=locations, - dating_label=dating_label, - dating_theso_huma_num_id=dating_theso_huma_num_id, - dating_theso_huma_num_parent_ids=dating_theso_huma_num_parent_ids, + **dating_dict, ) collections = [] diff --git a/lab/elasticsearch/documents.py b/lab/elasticsearch/documents.py index c2a30cdf2..5fcfc6c9b 100644 --- a/lab/elasticsearch/documents.py +++ b/lab/elasticsearch/documents.py @@ -24,7 +24,8 @@ class ObjectGroupDoc(os.InnerDoc): discovery_place_label = os.Text() collection = os.Keyword() inventory = os.Keyword() - dating_label = os.Text() + dating_period_label = os.Text() + dating_era_label = os.Text() objects = os.Object(ObjectDoc, multi=True) def add_object(self, label: str, collection: str, inventory: str): @@ -159,13 +160,17 @@ class Index: discovery_place_point = os.GeoPoint() collection = os.Text() inventory_number = os.Keyword() - dating_label = os.Text() - dating_theso_huma_num_id = os.Keyword() - dating_theso_huma_num_parent_ids = os.Keyword(multi=True) objects = os.Object(ObjectDoc, multi=True) collections = os.Keyword(multi=True) inventory_numbers = os.Keyword(multi=True) + dating_period_label = os.Text() + dating_period_theso_huma_num_id = os.Keyword() + dating_period_theso_huma_num_parent_ids = os.Keyword(multi=True) + dating_era_label = os.Text() + dating_era_theso_huma_num_id = os.Keyword() + dating_era_theso_huma_num_parent_ids = os.Keyword(multi=True) + def add_object( self, label: str, diff --git a/lab/elasticsearch/queries.py b/lab/elasticsearch/queries.py index db1222f39..d9bd0fada 100644 --- a/lab/elasticsearch/queries.py +++ b/lab/elasticsearch/queries.py @@ -102,8 +102,10 @@ def _match_param_to_query( return _status_query(value) case "materials": return _materials_query(value) - case "period_ids": - return _period_query(value) + case "dating_period_ids": + return _dating_period_query(value) + case "dating_era_ids": + return _dating_era_query(value) case "category": return _category_filter(value) case "c2rmf_id": @@ -192,8 +194,12 @@ def _materials_query(materials: list[str]): return _terms_query("materials", materials) -def _period_query(dating_ids: list[str]): - return _terms_query("dating_theso_huma_num_parent_ids", dating_ids) +def _dating_era_query(dating_ids: list[str]): + return _terms_query("dating_era_theso_huma_num_parent_ids", dating_ids) + + +def _dating_period_query(dating_ids: list[str]): + return _terms_query("dating_period_theso_huma_num_parent_ids", dating_ids) def _category_filter(category: Literal["project", "object"]): diff --git a/lab/elasticsearch/tests/_mock.py b/lab/elasticsearch/tests/_mock.py index f0d5e8590..39f4c9986 100644 --- a/lab/elasticsearch/tests/_mock.py +++ b/lab/elasticsearch/tests/_mock.py @@ -6,7 +6,8 @@ "q": "q", "status": "Status.DATA_AVAILABLE", "materials": ["material1", "material2"], - "period_ids": ["period1", "period2"], + "dating_period_ids": ["period1", "period2"], + "dating_era_ids": ["era1", "era2"], "category": "project", "c2rmf_id": "c2rmf_id", "created_from": datetime.datetime(2021, 1, 1).strftime("%Y-%m-%d"), @@ -32,33 +33,36 @@ { "multi_match": { "query": "q", - "fields": [ - "name", - "collections", - "collection", - "materials", - ], + "fields": ["name", "collections", "collection", "materials"], } }, {"term": {"status": "Status.DATA_AVAILABLE"}}, {"terms": {"materials": ["material1", "material2"]}}, - {"terms": {"dating_theso_huma_num_parent_ids": ["period1", "period2"]}}, + { + "terms": { + "dating_period_theso_huma_num_parent_ids": [ + "period1", + "period2", + ] + } + }, + {"terms": {"dating_era_theso_huma_num_parent_ids": ["era1", "era2"]}}, {"term": {"c2rmf_id": "c2rmf_id"}}, { "range": { "created": { "gte": "2021-01-01", - "lte": datetime.datetime.max, - }, - }, + "lte": datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), + } + } }, { "range": { "created": { - "gte": datetime.datetime.min, + "gte": datetime.datetime(1, 1, 1, 0, 0), "lte": "2021-12-31", - }, - }, + } + } }, { "geo_bounding_box": { diff --git a/lab/elasticsearch/tests/test_catalog.py b/lab/elasticsearch/tests/test_catalog.py index a218b90af..aee79fe1f 100644 --- a/lab/elasticsearch/tests/test_catalog.py +++ b/lab/elasticsearch/tests/test_catalog.py @@ -54,7 +54,8 @@ def test_build_project_catalog_document(): { "c2rmf_id": objectgroup.c2rmf_id, "collection": objectgroup.collection, - "dating_label": objectgroup.dating.label, + "dating_era_label": objectgroup.dating_era.label, + "dating_period_label": objectgroup.dating_period.label, "discovery_place_label": ( objectgroup.discovery_place.label if objectgroup.discovery_place @@ -81,10 +82,12 @@ def test_build_project_catalog_document(): @pytest.mark.django_db def test_build_object_group_catalog_document(): - dating = factories.PeriodFactory(theso_joconde_id=123) + dating_period = factories.PeriodFactory(concept_id=123) + dating_era = factories.EraFactory(concept_id=345) discovery_place = factories.LocationFactory() object_group = factories.ObjectGroupFactory( - dating=dating, + dating_period=dating_period, + dating_era=dating_era, discovery_place_location=discovery_place, inventory="123", ) @@ -95,17 +98,22 @@ def test_build_object_group_catalog_document(): run.run_object_groups.add(object_group) with mock.patch( - "lab.elasticsearch.catalog.fetch_epoques_parent_ids_from_id", + "lab.elasticsearch.catalog.fetch_period_parent_ids_from_id", return_value=[345, 567], - ) as fetch_epoques_mock: - document = build_object_group_catalog_document( - object_group=object_group, - runs=[run], - projects=[run.project], - is_data_available=True, - ) - - fetch_epoques_mock.assert_called_once_with(123) + ) as fetch_period_mock: + with mock.patch( + "lab.elasticsearch.catalog.fetch_era_parent_ids_from_id", + return_value=[890, 445], + ) as fetch_era_mock: + document = build_object_group_catalog_document( + object_group=object_group, + runs=[run], + projects=[run.project], + is_data_available=True, + ) + + fetch_period_mock.assert_called_once_with(123) + fetch_era_mock.assert_called_once_with(345) assert document.object_page_data.runs == [ { @@ -158,6 +166,10 @@ def test_build_object_group_catalog_document(): } ] - assert document.dating_label == dating.label - assert document.dating_theso_huma_num_id == 123 - assert document.dating_theso_huma_num_parent_ids == [345, 567] + assert document.dating_period_label == dating_period.label + assert document.dating_period_theso_huma_num_id == 123 + assert document.dating_period_theso_huma_num_parent_ids == [345, 567] + + assert document.dating_era_label == dating_era.label + assert document.dating_era_theso_huma_num_id == 345 + assert document.dating_era_theso_huma_num_parent_ids == [890, 445] diff --git a/lab/elasticsearch/tests/test_client.py b/lab/elasticsearch/tests/test_client.py index add03b7a4..0348c4f1a 100644 --- a/lab/elasticsearch/tests/test_client.py +++ b/lab/elasticsearch/tests/test_client.py @@ -70,10 +70,10 @@ def test_aggregate_date(catalog_client: CatalogClient): @pytest.mark.django_db def test_index_from_projects(catalog_client: CatalogClient): - dating = lab_factories.PeriodFactory(theso_joconde_id=123) + dating_period = lab_factories.PeriodFactory(concept_id=123) discovery_place = lab_factories.LocationFactory() object_group = lab_factories.ObjectGroupFactory( - dating=dating, + dating_period=dating_period, discovery_place_location=discovery_place, inventory="123", ) diff --git a/lab/migrations/0039_era_remove_period_period_unique_theso_joconde_id_and_more.py b/lab/migrations/0039_era_remove_period_period_unique_theso_joconde_id_and_more.py new file mode 100644 index 000000000..d51484e8d --- /dev/null +++ b/lab/migrations/0039_era_remove_period_period_unique_theso_joconde_id_and_more.py @@ -0,0 +1,160 @@ +# Generated by Django 5.0.6 on 2024-06-21 10:08 + +import django.db.models.deletion +from django.db import migrations, models + + +def move_period_to_era(apps, _): + """We have to move period to era because we saved Opentheso Humanum 'era' + with the field name 'period'.""" + period_model = apps.get_model("lab", "Period") + era_model = apps.get_model("lab", "Era") + objectgroup_model = apps.get_model("lab", "ObjectGroup") + for period in period_model.objects.all(): + era = era_model.objects.create( + label=period.label, concept_id=period.theso_joconde_id + ) + for og in objectgroup_model.objects.filter(dating=period): + og.dating_era = era + og.save() + period.delete() + + +def reverse_move_period_to_era(apps, _): + period_model = apps.get_model("lab", "Period") + era_model = apps.get_model("lab", "Era") + objectgroup_model = apps.get_model("lab", "ObjectGroup") + for era in era_model.objects.all(): + period = period_model.objects.create( + label=era.label, theso_joconde_id=era.concept_id + ) + for og in objectgroup_model.objects.filter(dating_era=era): + og.dating = period + og.save() + era.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("lab", "0038_location_unique_geonames_id"), + ] + + operations = [ + migrations.CreateModel( + name="Era", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "concept_id", + models.CharField( + blank=True, + max_length=255, + null=True, + verbose_name="Concept ID on Open Theso", + ), + ), + ("label", models.CharField(max_length=255, verbose_name="Label")), + ], + options={ + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="era", + constraint=models.UniqueConstraint( + fields=("label", "concept_id"), + name="era_thesorus_concept_unique_label_concept_id", + ), + ), + migrations.AddConstraint( + model_name="era", + constraint=models.UniqueConstraint( + fields=("concept_id",), name="era_thesorus_concept_unique_concept_id" + ), + ), + migrations.AddField( + model_name="objectgroup", + name="dating_era", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="lab.era", + verbose_name="Era", + ), + ), + migrations.RunSQL( + "SET CONSTRAINTS ALL IMMEDIATE;", "SET CONSTRAINTS ALL DEFERRED;" + ), + migrations.RunPython( + move_period_to_era, + reverse_move_period_to_era, + ), + migrations.RunSQL( + "SET CONSTRAINTS ALL DEFERRED;", "SET CONSTRAINTS ALL IMMEDIATE;" + ), + migrations.RemoveConstraint( + model_name="period", + name="period_unique_theso_joconde_id", + ), + migrations.RemoveConstraint( + model_name="period", + name="period_unique_label_theso_joconde_id", + ), + migrations.RemoveField( + model_name="objectgroup", + name="dating", + ), + migrations.RemoveField( + model_name="period", + name="theso_joconde_id", + ), + migrations.AddField( + model_name="objectgroup", + name="dating_period", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="lab.period", + verbose_name="Period", + ), + ), + migrations.AddField( + model_name="period", + name="concept_id", + field=models.CharField( + blank=True, + max_length=255, + null=True, + verbose_name="Concept ID on Open Theso", + ), + ), + migrations.AlterField( + model_name="period", + name="label", + field=models.CharField(max_length=255, verbose_name="Label"), + ), + migrations.AddConstraint( + model_name="period", + constraint=models.UniqueConstraint( + fields=("label", "concept_id"), + name="period_thesorus_concept_unique_label_concept_id", + ), + ), + migrations.AddConstraint( + model_name="period", + constraint=models.UniqueConstraint( + fields=("concept_id",), name="period_thesorus_concept_unique_concept_id" + ), + ), + ] diff --git a/lab/objects/assets/js/services.ts b/lab/objects/assets/js/services.ts index d2e4d47a5..0380dfc0c 100644 --- a/lab/objects/assets/js/services.ts +++ b/lab/objects/assets/js/services.ts @@ -5,7 +5,7 @@ interface ObjectGroupResponseElement { id: number; label: string; object_count: number; - dating: string; + dating: string; // this is linked to Django ObjectGroup.dating_era materials: string[]; } @@ -32,6 +32,9 @@ export async function fetchRunObjectGroups( })); } +/** + * Fetches the object groups available to import for a run. + */ export async function fetchAvailableObjectGroups( runId: string, ): Promise { diff --git a/lab/objects/c2rmf.py b/lab/objects/c2rmf.py index b6b8c60fe..03c80f777 100644 --- a/lab/objects/c2rmf.py +++ b/lab/objects/c2rmf.py @@ -4,7 +4,9 @@ import requests -from ..models import ObjectGroup, Period +from lab.thesauri.models import Era + +from ..models import ObjectGroup class ErosHTTPError(requests.RequestException): @@ -90,7 +92,7 @@ def fetch_full_objectgroup_from_eros( updated_og.label = data["title"] if data.get("dtfrom") or data.get("period"): dating_label = data.get("dtfrom") or data["period"] - updated_og.dating = Period(label=dating_label) + updated_og.dating_era = Era(label=dating_label) updated_og.collection = data.get("collection") or "" updated_og.inventory = data.get("inv") or "" updated_og.materials = (data.get("support") or "").split(" / ") diff --git a/lab/objects/forms.py b/lab/objects/forms.py index 57a32bad2..e2b12ffcd 100644 --- a/lab/objects/forms.py +++ b/lab/objects/forms.py @@ -9,7 +9,7 @@ from . import widgets from .c2rmf import ErosHTTPError, fetch_partial_objectgroup_from_eros -from .models import Location, ObjectGroup, Period +from .models import Era, Location, ObjectGroup, Period logger = logging.getLogger(__name__) @@ -39,7 +39,8 @@ class Meta: "add_type", "label", "object_count", - "dating", + "dating_era", + "dating_period", "materials", "discovery_place_location", "inventory", @@ -51,20 +52,22 @@ class Meta: Click on suggestion or add a comma to add to the list." ), "discovery_place_location": _("Start typing to search for a location"), - "dating": _("Start typing to search for a period"), + "dating_period": _("Start typing to search for a period"), } widgets = { "materials": TagsInput(), "discovery_place_location": widgets.LocationAutoCompleteWidget(), - "dating": widgets.DatingAutoCompleteWidget(), + "dating_period": widgets.PeriodDatingAutoCompleteWidget(), + "dating_era": widgets.EraDatingAutoCompleteWidget(), } def __init__(self, *args, instance: ObjectGroup | None = None, **kwargs): super().__init__(*args, **kwargs, instance=instance) # type: ignore[misc] # We must check if attribute is in self.fields because it can be removed # in admin view when page is readlonly. - if "dating" in self.fields: - self.fields["dating"].required = True + for field_name in ["dating_period", "dating_era"]: + if field_name in self.fields: + self.fields[field_name].required = True # Set object count initial value if "object_count" in self.fields: self.fields["object_count"].initial = 1 @@ -82,11 +85,12 @@ def __init__(self, *args, instance: ObjectGroup | None = None, **kwargs): self.fields["discovery_place_location"].widget.instance = ( instance.discovery_place_location ) - self.fields["dating"].widget.instance = instance.dating + self.fields["dating_period"].widget.instance = instance.dating_period + self.fields["dating_era"].widget.instance = instance.dating_era def full_clean(self): self.try_populate_discovery_place_location() - self.try_populate_dating() + self.try_populate_dating_models() return super().full_clean() def try_populate_discovery_place_location(self): @@ -116,16 +120,17 @@ def try_populate_discovery_place_location(self): ) # make a copy of the data because self.data is immutable self.data["discovery_place_location"] = location.pk - def try_populate_dating(self): - if not self.data.get("dating") and self.data.get("dating__label"): - period, _ = Period.objects.get_or_create( - label=self.data["dating__label"], - theso_joconde_id=self.data.get("dating__theso_joconde_id"), - ) - self.data = ( - self.data.copy() - ) # make a copy of the data because self.data is immutable - self.data["dating"] = period.pk + def try_populate_dating_models(self): + for field_name, theso_model in [("dating_period", Period), ("dating_era", Era)]: + if not self.data.get(field_name) and self.data.get(f"{field_name}__label"): + period, _ = theso_model.objects.get_or_create( + label=self.data[f"{field_name}__label"], + concept_id=self.data.get(f"{field_name}__concept_id"), + ) + self.data = ( + self.data.copy() + ) # make a copy of the data because self.data is immutable + self.data[field_name] = period.pk def is_multipart(self) -> Any: if not self.instance.id: @@ -197,7 +202,8 @@ class Meta: "c2rmf_id", "label", "object_count", - "dating", + "dating_era", + "dating_period", "materials", "discovery_place_location", "inventory", diff --git a/lab/objects/models.py b/lab/objects/models.py index d01430fe5..19ace09d2 100644 --- a/lab/objects/models.py +++ b/lab/objects/models.py @@ -4,27 +4,7 @@ from shared.models import TimestampedModel - -class Period(models.Model): - label = models.CharField(_("Name"), max_length=255) - - theso_joconde_id = models.CharField( - "Joconde Thesorus ID", max_length=255, null=True, blank=True - ) - - def __str__(self) -> str: - return str(self.label) - - class Meta: - constraints = [ - models.UniqueConstraint( - fields=["label", "theso_joconde_id"], - name="period_unique_label_theso_joconde_id", - ), - models.UniqueConstraint( - fields=["theso_joconde_id"], name="period_unique_theso_joconde_id" - ), - ] +from ..thesauri.models import Era, Period class Location(models.Model): @@ -66,10 +46,17 @@ class ObjectGroup(TimestampedModel): max_length=255, blank=True, ) - dating = models.ForeignKey( + dating_period = models.ForeignKey( Period, on_delete=models.SET_NULL, - verbose_name=_("Dating"), + verbose_name=_("Period"), + blank=True, + null=True, + ) + dating_era = models.ForeignKey( + Era, + on_delete=models.SET_NULL, + verbose_name=_("Era"), blank=True, null=True, ) @@ -104,9 +91,10 @@ def __str__(self) -> str: "object_count": self.object_count } label = "({}) {}".format(count_str, label) - materials = ", ".join(self.materials) - - return f"{label} - {self.dating} - {materials}" + if self.materials: + materials = ", ".join(self.materials) + label = f"{label} - {materials}" + return label class Meta: verbose_name = _("Object / Sample") diff --git a/lab/objects/tests/forms/test_objectgroup_form.py b/lab/objects/tests/forms/test_objectgroup_form.py index b335c72e1..d605b7b78 100644 --- a/lab/objects/tests/forms/test_objectgroup_form.py +++ b/lab/objects/tests/forms/test_objectgroup_form.py @@ -1,6 +1,8 @@ import pytest from django.forms import widgets +from lab.thesauri.models import Era + from ...forms import ObjectGroupAddChoices, ObjectGroupForm from ...models import Location, ObjectGroup, Period @@ -120,31 +122,46 @@ def test_try_populate_discovery_place_location_updates_lat_and_long(): @pytest.mark.django_db def test_try_populate_dating_create_period(): label = "Moyen âge" - theso_joconde_id = 1234 + concept_id = 1234 form = ObjectGroupForm() form.data = { - "dating__label": label, - "dating__theso_joconde_id": theso_joconde_id, + "dating_period__label": label, + "dating_period__concept_id": concept_id, + } + + form.try_populate_dating_models() + + dating_period = Period.objects.get(label=label, concept_id=concept_id) + assert form.data["dating_period"] == dating_period.pk + + +@pytest.mark.django_db +def test_try_populate_dating_find_period(): + dating = Period.objects.create(label="Moyen âge", concept_id=1234) + + form = ObjectGroupForm() + form.data = { + "dating_period__label": "Moyen âge", + "dating_period__concept_id": 1234, } - form.try_populate_dating() + form.try_populate_dating_models() - dating = Period.objects.get(label=label, theso_joconde_id=theso_joconde_id) - assert form.data["dating"] == dating.pk + assert form.data["dating_period"] == dating.pk @pytest.mark.django_db -def test_try_populate_dating_find_dating(): - dating = Period.objects.create(label="Moyen âge", theso_joconde_id=1234) +def test_try_populate_dating_find_era(): + dating = Era.objects.create(label="IIe siècle", concept_id=1234) form = ObjectGroupForm() form.data = { - "dating__label": "Moyen âge", - "dating__theso_joconde_id": 1234, + "dating_era__label": "IIe siècle", + "dating_era__concept_id": 1234, } - form.try_populate_dating() + form.try_populate_dating_models() - assert form.data["dating"] == dating.pk + assert form.data["dating_era"] == dating.pk diff --git a/lab/objects/tests/test_c2mrf.py b/lab/objects/tests/test_c2mrf.py index da28ebe0f..661a29c30 100644 --- a/lab/objects/tests/test_c2mrf.py +++ b/lab/objects/tests/test_c2mrf.py @@ -39,7 +39,7 @@ def test_fetch_full_objectgroup_from_eros(_): assert og.object_count == 1 assert og.c2rmf_id == "C2RMF65980" assert og.label == "Majolique" - assert isinstance(og.dating, Period) - assert og.dating.label == "1500" + assert isinstance(og.dating_period, Period) + assert og.dating_period.label == "1500" assert og.inventory == "ODUT 01107" assert og.materials == ["terre cuite"] diff --git a/lab/objects/tests/test_object_formset.py b/lab/objects/tests/test_object_formset.py index 6c25e0dfe..78f97a428 100644 --- a/lab/objects/tests/test_object_formset.py +++ b/lab/objects/tests/test_object_formset.py @@ -147,7 +147,7 @@ def setUp(self): self.inline = ObjectInline(ObjectGroup, admin_site=AdminSite()) self.objectgroup = ObjectGroup( label="Object group", - dating=Period.objects.get_or_create(label="XIXe")[0], + dating_period=Period.objects.get_or_create(label="XIXe")[0], materials=["wood"], object_count=0, ) diff --git a/lab/objects/widgets.py b/lab/objects/widgets.py index d95c40935..e1a1dd7f3 100644 --- a/lab/objects/widgets.py +++ b/lab/objects/widgets.py @@ -2,9 +2,10 @@ from django.forms import widgets +from lab.thesauri.models import ThesorusConceptModel from lab.widgets import AutoCompleteWidget -from .models import Location, Period +from .models import Era, Location, Period class ImportFromInput(widgets.TextInput): @@ -68,13 +69,41 @@ class Media: class DatingAutoCompleteWidget(AutoCompleteWidget): + model: type[ThesorusConceptModel] | None = None + template_name = "widgets/dating_autocomplete_widget.html" + def get_context( + self, name: str, value: Any, attrs: dict[str, Any] | None + ) -> dict[str, Any]: + context = super().get_context(name, value, attrs) + if self.model is None: + raise ValueError("model is required") + context["widget"]["field_name"] = self.model.__name__.lower() + # pylint: disable=no-member + context["widget"]["opentheso_theso_id"] = self.model.OPENTHESO_THESO_ID + return context + + +class PeriodDatingAutoCompleteWidget(DatingAutoCompleteWidget): model = Period + typeahead_list_webcomponent_name = "period-type-ahead" class Media: js = ( - "web-components/period-type-ahead.js", + "web-components/dating-open-theso-type-ahead.js", "js/widgets/dating-autocomplete-widget.js", ) css = {"all": ("css/widgets/autocomplete-widget.css",)} + + +class EraDatingAutoCompleteWidget(DatingAutoCompleteWidget): + model = Era + typeahead_list_webcomponent_name = "era-type-ahead" + + class Media: + js = ( + "web-components/dating-open-theso-type-ahead.js", + "js/widgets/era-autocomplete-widget.js", + ) + css = {"all": ("css/widgets/autocomplete-widget.css",)} diff --git a/lab/static/css/admin/objectgroup.css b/lab/static/css/admin/objectgroup.css index fad6e6b93..59551b24e 100644 --- a/lab/static/css/admin/objectgroup.css +++ b/lab/static/css/admin/objectgroup.css @@ -34,7 +34,7 @@ input[type="submit"][disabled].default { margin-bottom: 1rem; } -.field-discovery_place_location, .field-dating, .field-materials { +.field-discovery_place_location, .field-dating_period, .field-dating_era, .field-materials { overflow: initial; } diff --git a/lab/static/js/widgets/dating-autocomplete-widget.js b/lab/static/js/widgets/dating-autocomplete-widget.js index 748ee0592..0ef5d0c0d 100644 --- a/lab/static/js/widgets/dating-autocomplete-widget.js +++ b/lab/static/js/widgets/dating-autocomplete-widget.js @@ -8,7 +8,7 @@ const parentField = event.target.closest(baseSelector); parentField.querySelector(`${baseSelector}__label`).value = label; - parentField.querySelector(`${baseSelector}__theso_joconde_id`).value = id; + parentField.querySelector(`${baseSelector}__concept_id`).value = id; parentField.querySelector(".typeahead-list").classList.add("hidden"); } @@ -19,21 +19,20 @@ const idInput = datingField.querySelector(`${baseSelector}__id`); if (idInput.value && idInput.value !== "") { idInput.value = ""; - datingField.querySelector(`${baseSelector}__theso_joconde_id`).value = ""; + datingField.querySelector(`${baseSelector}__concept_id`).value = ""; } } document.addEventListener("DOMContentLoaded", function () { - // Add event listeners to institution input elements + // Add event listeners to dating input elements document.querySelectorAll(`${baseSelector}`).forEach((el) => { el.querySelector(`${baseSelector}__label`).addEventListener( "input", onInput, ); - el.querySelector("div[is='period-type-ahead']").addEventListener( - "result-click", - onResultClicked, - ); + el.querySelector( + "div[is='dating-open-theso-type-ahead']", + ).addEventListener("result-click", onResultClicked); }); }); })(); diff --git a/lab/templates/widgets/dating_autocomplete_widget.html b/lab/templates/widgets/dating_autocomplete_widget.html index 5aeb0aa46..038f08b8d 100644 --- a/lab/templates/widgets/dating_autocomplete_widget.html +++ b/lab/templates/widgets/dating_autocomplete_widget.html @@ -1,9 +1,34 @@
- -
+
+ +
+
+
- - + +
\ No newline at end of file diff --git a/lab/tests/factories.py b/lab/tests/factories.py index 150614906..8ebac4fc9 100644 --- a/lab/tests/factories.py +++ b/lab/tests/factories.py @@ -4,6 +4,8 @@ import factory.fuzzy from django.contrib.auth import get_user_model +from lab.thesauri.models import Era + from ..models import Object, ObjectGroup, Participation, Period, Project, Run from ..objects.models import Location @@ -121,6 +123,13 @@ class Meta: label = factory.Faker("date") +class EraFactory(factory.django.DjangoModelFactory): + class Meta: + model = Era + + label = factory.Faker("date") + + class ObjectFactory(factory.django.DjangoModelFactory): class Meta: model = Object @@ -133,7 +142,8 @@ class Meta: model = ObjectGroup label = factory.Faker("name") - dating = factory.SubFactory(PeriodFactory) + dating_period = factory.SubFactory(PeriodFactory) + dating_era = factory.SubFactory(EraFactory) materials = factory.fuzzy.FuzzyChoice(["wood", "stone", "glass", "metal"], list) object_count = 3 diff --git a/lab/tests/test_opentheso.py b/lab/tests/test_opentheso.py index 8ccffd6b7..d3b0d5663 100644 --- a/lab/tests/test_opentheso.py +++ b/lab/tests/test_opentheso.py @@ -1,9 +1,9 @@ from unittest import mock -from ..opentheso import fetch_parent_ids_from_id +from ..thesauri.opentheso import fetch_parent_ids_from_id -@mock.patch("lab.opentheso.requests") +@mock.patch("lab.thesauri.opentheso.requests") def test_fetch_parent_ids_from_id(request_mock: mock.MagicMock): response_data = { "ELEMENT/IGNORED_ELEMENT": {}, diff --git a/lab/thesauri/__init__.py b/lab/thesauri/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lab/thesauri/models.py b/lab/thesauri/models.py new file mode 100644 index 000000000..0f26034a7 --- /dev/null +++ b/lab/thesauri/models.py @@ -0,0 +1,40 @@ +from django.db import models + + +class ThesorusConceptModel(models.Model): + """Abstract model for concept in thesauri.""" + + OPENTHESO_THESO_ID: str | None = None + + concept_id = models.CharField( + "Concept ID on Open Theso", + max_length=255, + null=True, + blank=True, + ) + label = models.CharField("Label", max_length=255) + + class Meta: + abstract = True + + constraints = [ + models.UniqueConstraint( + fields=["label", "concept_id"], + name="%(class)s_thesorus_concept_unique_label_concept_id", + ), + models.UniqueConstraint( + fields=["concept_id"], + name="%(class)s_thesorus_concept_unique_concept_id", + ), + ] + + def __str__(self) -> str: + return str(self.label) + + +class Period(ThesorusConceptModel): + OPENTHESO_THESO_ID = "th287" + + +class Era(ThesorusConceptModel): + OPENTHESO_THESO_ID = "th289" diff --git a/lab/opentheso.py b/lab/thesauri/opentheso.py similarity index 79% rename from lab/opentheso.py rename to lab/thesauri/opentheso.py index 741ad91a5..258a7be95 100644 --- a/lab/opentheso.py +++ b/lab/thesauri/opentheso.py @@ -3,6 +3,8 @@ import requests +from .models import Era, Period + logger = logging.getLogger(__name__) THESO_IDS = {"epoques": "th289"} @@ -36,5 +38,9 @@ def fetch_parent_ids_from_id(theso_id: str, concept_id: str) -> list[str]: return [] -def fetch_epoques_parent_ids_from_id(concept_id: str): - return fetch_parent_ids_from_id(THESO_IDS["epoques"], concept_id) +def fetch_era_parent_ids_from_id(concept_id: str): + return fetch_parent_ids_from_id(Era.OPENTHESO_THESO_ID, concept_id) + + +def fetch_period_parent_ids_from_id(concept_id: str): + return fetch_parent_ids_from_id(Period.OPENTHESO_THESO_ID, concept_id) diff --git a/pylint.txt b/pylint.txt new file mode 100644 index 000000000..e69de29bb