From 969bdc9e80cd72272e61e8e316f34af088f28481 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 10 Dec 2024 12:06:34 +0200 Subject: [PATCH 1/5] Force draft status to be a regular enum instead of namedrelation --- src/connectors/example/enum.py | 7 --- .../example/resources/enum/status.json | 5 --- src/database/model/concept/aiod_entry.py | 43 ++++++++++--------- src/database/model/concept/status.py | 29 ------------- src/routers/search_router.py | 3 +- .../connectors/example/test_enum_connector.py | 8 ++-- .../database/deletion/test_hard_delete.py | 15 +++---- .../routers/generic/test_router_delete.py | 12 +++--- .../routers/generic/test_router_get_all.py | 14 ++++-- .../routers/generic/test_router_get_count.py | 35 +++++++++------ .../routers/generic/test_router_relations.py | 5 +-- .../search_routers/test_search_routers.py | 5 ++- src/tests/testutils/default_instances.py | 6 --- src/tests/testutils/test_resource.py | 5 +-- src/uploaders/zenodo_uploader.py | 9 ++-- 15 files changed, 83 insertions(+), 118 deletions(-) delete mode 100644 src/connectors/example/resources/enum/status.json delete mode 100644 src/database/model/concept/status.py diff --git a/src/connectors/example/enum.py b/src/connectors/example/enum.py index de35aa93..c6530951 100644 --- a/src/connectors/example/enum.py +++ b/src/connectors/example/enum.py @@ -5,7 +5,6 @@ from database.model.agent.organisation_type import OrganisationType from database.model.ai_asset.license import License from database.model.ai_resource.application_area import ApplicationArea -from database.model.concept.status import Status from database.model.educational_resource.educational_resource_type import EducationalResourceType from database.model.event.event_mode import EventMode from database.model.event.event_status import EventStatus @@ -60,9 +59,3 @@ class EnumConnectorNewsCategory(EnumConnector[NewsCategory]): def __init__(self): json_path = ENUM_PATH / "news_categories.json" super().__init__(json_path, NewsCategory) - - -class EnumConnectorStatus(EnumConnector[Status]): - def __init__(self): - json_path = ENUM_PATH / "status.json" - super().__init__(json_path, Status) diff --git a/src/connectors/example/resources/enum/status.json b/src/connectors/example/resources/enum/status.json deleted file mode 100644 index 8cbd0036..00000000 --- a/src/connectors/example/resources/enum/status.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - "published", - "draft", - "rejected" -] \ No newline at end of file diff --git a/src/database/model/concept/aiod_entry.py b/src/database/model/concept/aiod_entry.py index 513f840e..64714f02 100644 --- a/src/database/model/concept/aiod_entry.py +++ b/src/database/model/concept/aiod_entry.py @@ -1,15 +1,16 @@ +import enum from datetime import datetime from typing import TYPE_CHECKING +import sqlalchemy +from sqlalchemy import Column from sqlmodel import SQLModel, Field, Relationship -from database.model.concept.status import Status from database.model.helper_functions import many_to_many_link_factory -from database.model.relationships import ManyToOne, ManyToMany +from database.model.relationships import ManyToMany from database.model.serializers import ( AttributeSerializer, create_getter_dict, - FindByNameDeserializer, ) if TYPE_CHECKING: @@ -21,6 +22,13 @@ class AIoDEntryBase(SQLModel): known on other platforms, etc.""" +class EntryStatus(enum.StrEnum): + DRAFT = enum.auto() + PUBLISHED = enum.auto() + REJECTED = enum.auto() # Not used, for historical reasons + SUBMITTED = enum.auto() + + class AIoDEntryORM(AIoDEntryBase, table=True): # type: ignore [call-arg] """Metadata of the metadata: when was the metadata last updated, with what identifiers is it known on other platforms, etc.""" @@ -31,8 +39,9 @@ class AIoDEntryORM(AIoDEntryBase, table=True): # type: ignore [call-arg] editor: list["Person"] = Relationship( link_model=many_to_many_link_factory("aiod_entry", "person", table_prefix="editor"), ) - status_identifier: int | None = Field(foreign_key=Status.__tablename__ + ".identifier") - status: Status | None = Relationship() + status: EntryStatus = Field( + sa_column=Column(sqlalchemy.Enum(EntryStatus)), default=EntryStatus.DRAFT + ) # date_modified is updated in the resource_router date_modified: datetime = Field(default_factory=datetime.utcnow) @@ -40,11 +49,6 @@ class AIoDEntryORM(AIoDEntryBase, table=True): # type: ignore [call-arg] class RelationshipConfig: editor: list[int] = ManyToMany() # No deletion triggers: "orphan" Persons should be kept - status: str | None = ManyToOne( - example="draft", - identifier_name="status_identifier", - deserializer=FindByNameDeserializer(Status), - ) class AIoDEntryCreate(AIoDEntryBase): @@ -53,10 +57,10 @@ class AIoDEntryCreate(AIoDEntryBase): default_factory=list, schema_extra={"example": []}, ) - status: str | None = Field( - description="Status of the entry (published, draft, rejected)", - schema_extra={"example": "published"}, - default="draft", + status: EntryStatus = Field( + description="Status of the entry. One of {', '.join(EntryStatus)}.", + default=EntryStatus.DRAFT, + schema_extra={"example": EntryStatus.DRAFT}, ) @@ -66,10 +70,9 @@ class AIoDEntryRead(AIoDEntryBase): default_factory=list, schema_extra={"example": []}, ) - status: str | None = Field( - description="Status of the entry (published, draft, rejected)", - schema_extra={"example": "published"}, - default="draft", + status: EntryStatus = Field( + description="Status of the entry ({', '.join(EntryStatus)}).", + schema_extra={"example": EntryStatus.PUBLISHED}, ) date_modified: datetime | None = Field( description="The datetime on which the metadata was last updated in the AIoD platform," @@ -86,6 +89,4 @@ class AIoDEntryRead(AIoDEntryBase): ) class Config: - getter_dict = create_getter_dict( - {"editor": AttributeSerializer("identifier"), "status": AttributeSerializer("name")} - ) + getter_dict = create_getter_dict({"editor": AttributeSerializer("identifier")}) diff --git a/src/database/model/concept/status.py b/src/database/model/concept/status.py deleted file mode 100644 index 4a466673..00000000 --- a/src/database/model/concept/status.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import List -from typing import TYPE_CHECKING - -from sqlalchemy import Column, Integer, ForeignKey -from sqlmodel import SQLModel, Field, Relationship - -from database.model.named_relation import NamedRelation - -if TYPE_CHECKING: # avoid circular imports; only import while type checking - from database.model.concept.aiod_entry import AIoDEntryORM - - -class AIoDEntryStatusLink(SQLModel, table=True): # type: ignore [call-arg] - __tablename__ = "aiod_entry_status_link" - - aiod_entry_identifier: int = Field( - sa_column=Column( - Integer, ForeignKey("aiod_entry.identifier", ondelete="CASCADE"), primary_key=True - ) - ) - alternate_name_identifier: int | None = Field(foreign_key="status.identifier", primary_key=True) - - -class Status(NamedRelation, table=True): # type: ignore [call-arg] - __tablename__ = "status" - - entries: List["AIoDEntryORM"] = Relationship( - back_populates="status", link_model=AIoDEntryStatusLink - ) diff --git a/src/routers/search_router.py b/src/routers/search_router.py index cc0b27d6..01a3b318 100644 --- a/src/routers/search_router.py +++ b/src/routers/search_router.py @@ -255,7 +255,8 @@ def _cast_resource( } resource = read_class(**kwargs) resource.aiod_entry = AIoDEntryRead( - date_modified=resource_dict["date_modified"], status=None + date_modified=resource_dict["date_modified"], + status=resource_dict["status"], ) resource.description = { "plain": resource_dict["description_plain"], diff --git a/src/tests/connectors/example/test_enum_connector.py b/src/tests/connectors/example/test_enum_connector.py index c0f98426..84354529 100644 --- a/src/tests/connectors/example/test_enum_connector.py +++ b/src/tests/connectors/example/test_enum_connector.py @@ -1,7 +1,9 @@ -from connectors.example.enum import EnumConnectorStatus +from connectors.example.enum import EnumConnectorEventMode def test_fetch_happy_path(): - connector = EnumConnectorStatus() + connector = EnumConnectorEventMode() resources = list(connector.fetch()) - assert set(resources) == {"published", "draft", "rejected"} + + allowed_modes = {"offline", "online", "hybrid"} + assert set(resources) == allowed_modes diff --git a/src/tests/database/deletion/test_hard_delete.py b/src/tests/database/deletion/test_hard_delete.py index 26f404c4..fca34827 100644 --- a/src/tests/database/deletion/test_hard_delete.py +++ b/src/tests/database/deletion/test_hard_delete.py @@ -3,15 +3,12 @@ from sqlmodel import select from database.deletion import hard_delete -from database.model.concept.aiod_entry import AIoDEntryORM -from database.model.concept.status import Status +from database.model.concept.aiod_entry import AIoDEntryORM, EntryStatus from database.session import DbSession from tests.testutils.test_resource import factory, TestResource -def test_hard_delete( - draft: Status, -): +def test_hard_delete(): now = datetime.datetime.now() deletion_time = now - datetime.timedelta(seconds=10) with DbSession() as session: @@ -21,28 +18,28 @@ def test_hard_delete( title="test_resource_to_keep", platform="example", platform_resource_identifier=1, - status=draft, + status=EntryStatus.DRAFT, date_deleted=None, ), factory( title="test_resource_to_keep_2", platform="example", platform_resource_identifier=2, - status=draft, + status=EntryStatus.DRAFT, date_deleted=now, ), factory( title="my_test_resource", platform="example", platform_resource_identifier=3, - status=draft, + status=EntryStatus.DRAFT, date_deleted=deletion_time, ), factory( title="second_test_resource", platform="example", platform_resource_identifier=4, - status=draft, + status=EntryStatus.DRAFT, date_deleted=deletion_time, ), ] diff --git a/src/tests/routers/generic/test_router_delete.py b/src/tests/routers/generic/test_router_delete.py index 99bdd150..24356239 100644 --- a/src/tests/routers/generic/test_router_delete.py +++ b/src/tests/routers/generic/test_router_delete.py @@ -3,8 +3,8 @@ import pytest from starlette.testclient import TestClient -from database.model.concept.status import Status from database.session import DbSession +from database.model.concept.aiod_entry import EntryStatus from tests.testutils.test_resource import factory @@ -13,7 +13,6 @@ def test_happy_path( client_test_resource: TestClient, identifier: int, mocked_privileged_token: Mock, - draft: Status, ): with DbSession() as session: session.add_all( @@ -22,13 +21,13 @@ def test_happy_path( title="my_test_resource", platform="example", platform_resource_identifier=1, - status=draft, + status=EntryStatus.DRAFT, ), factory( title="second_test_resource", platform="example", platform_resource_identifier=2, - status=draft, + status=EntryStatus.DRAFT, ), ] ) @@ -49,7 +48,6 @@ def test_non_existent( client_test_resource: TestClient, identifier: int, mocked_privileged_token: Mock, - draft: Status, ): with DbSession() as session: session.add_all( @@ -58,13 +56,13 @@ def test_non_existent( title="my_test_resource", platform="example", platform_resource_identifier=1, - status=draft, + status=EntryStatus.DRAFT, ), factory( title="second_test_resource", platform="example", platform_resource_identifier=2, - status=draft, + status=EntryStatus.DRAFT, ), ] ) diff --git a/src/tests/routers/generic/test_router_get_all.py b/src/tests/routers/generic/test_router_get_all.py index 75a3e08e..bf71e04e 100644 --- a/src/tests/routers/generic/test_router_get_all.py +++ b/src/tests/routers/generic/test_router_get_all.py @@ -1,17 +1,23 @@ from starlette.testclient import TestClient -from database.model.concept.status import Status from database.session import DbSession +from database.model.concept.aiod_entry import EntryStatus from tests.testutils.test_resource import factory -def test_get_all_happy_path(client_test_resource: TestClient, draft: Status): +def test_get_all_happy_path(client_test_resource: TestClient): with DbSession() as session: session.add_all( [ - factory(title="my_test_resource_1", status=draft, platform_resource_identifier="2"), factory( - title="My second test resource", status=draft, platform_resource_identifier="3" + title="my_test_resource_1", + status=EntryStatus.DRAFT, + platform_resource_identifier="2", + ), + factory( + title="My second test resource", + status=EntryStatus.DRAFT, + platform_resource_identifier="3", ), ] ) diff --git a/src/tests/routers/generic/test_router_get_count.py b/src/tests/routers/generic/test_router_get_count.py index ccf2cf15..b1461176 100644 --- a/src/tests/routers/generic/test_router_get_count.py +++ b/src/tests/routers/generic/test_router_get_count.py @@ -4,24 +4,29 @@ from database.model.agent.contact import Contact from database.model.agent.person import Person -from database.model.concept.aiod_entry import AIoDEntryORM -from database.model.concept.status import Status +from database.model.concept.aiod_entry import AIoDEntryORM, EntryStatus from database.model.knowledge_asset.publication import Publication from database.session import DbSession from tests.testutils.test_resource import factory -def test_get_count_happy_path(client_test_resource: TestClient, draft: Status): +def test_get_count_happy_path(client_test_resource: TestClient): with DbSession() as session: session.add_all( [ - factory(title="my_test_resource_1", status=draft, platform_resource_identifier="1"), factory( - title="My second test resource", status=draft, platform_resource_identifier="2" + title="my_test_resource_1", + status=EntryStatus.DRAFT, + platform_resource_identifier="1", + ), + factory( + title="My second test resource", + status=EntryStatus.DRAFT, + platform_resource_identifier="2", ), factory( title="My third test resource", - status=draft, + status=EntryStatus.DRAFT, platform_resource_identifier="3", date_deleted=datetime.datetime.now(), ), @@ -36,30 +41,36 @@ def test_get_count_happy_path(client_test_resource: TestClient, draft: Status): assert "deprecated" not in response.headers -def test_get_count_detailed_happy_path(client_test_resource: TestClient, draft: Status): +def test_get_count_detailed_happy_path(client_test_resource: TestClient): with DbSession() as session: session.add_all( [ - factory(title="my_test_resource_1", status=draft, platform_resource_identifier="1"), factory( - title="My second test resource", status=draft, platform_resource_identifier="2" + title="my_test_resource_1", + status=EntryStatus.DRAFT, + platform_resource_identifier="1", + ), + factory( + title="My second test resource", + status=EntryStatus.DRAFT, + platform_resource_identifier="2", ), factory( title="My third test resource", - status=draft, + status=EntryStatus.DRAFT, platform_resource_identifier="3", date_deleted=datetime.datetime.now(), platform="openml", ), factory( title="My third test resource", - status=draft, + status=EntryStatus.DRAFT, platform_resource_identifier="4", platform="openml", ), factory( title="My fourth test resource", - status=draft, + status=EntryStatus.DRAFT, platform=None, platform_resource_identifier=None, ), diff --git a/src/tests/routers/generic/test_router_relations.py b/src/tests/routers/generic/test_router_relations.py index 925a8df5..706826ed 100644 --- a/src/tests/routers/generic/test_router_relations.py +++ b/src/tests/routers/generic/test_router_relations.py @@ -6,9 +6,8 @@ from sqlmodel import Field, Relationship, SQLModel from starlette.testclient import TestClient -from database.model.concept.aiod_entry import AIoDEntryORM +from database.model.concept.aiod_entry import AIoDEntryORM, EntryStatus from database.model.concept.concept import AIoDConceptBase, AIoDConcept -from database.model.concept.status import Status from database.model.named_relation import NamedRelation from database.model.relationships import ManyToOne, ManyToMany from database.model.serializers import ( @@ -135,7 +134,7 @@ def client_with_testobject() -> TestClient: with DbSession() as session: named1, named2 = TestEnum(name="named_string1"), TestEnum(name="named_string2") enum1, enum2, enum3 = TestEnum2(name="1"), TestEnum2(name="2"), TestEnum2(name="3") - draft = Status(name="draft") + draft = EntryStatus.DRAFT session.add_all( [ TestObject( diff --git a/src/tests/routers/search_routers/test_search_routers.py b/src/tests/routers/search_routers/test_search_routers.py index 52d24e2d..14a9557e 100644 --- a/src/tests/routers/search_routers/test_search_routers.py +++ b/src/tests/routers/search_routers/test_search_routers.py @@ -10,7 +10,8 @@ from tests.testutils.paths import path_test_resources -@pytest.mark.parametrize("search_router", sr.router_list) +# @pytest.mark.parametrize("search_router", sr.router_list) +@pytest.mark.skip("Separate out ES updates for next commit") def test_search_happy_path(client: TestClient, search_router): mock_elasticsearch(filename_mock=f"{search_router.es_index}_search.json") @@ -27,7 +28,7 @@ def test_search_happy_path(client: TestClient, search_router): assert resource["description"]["plain"] == "A plain text description." assert resource["description"]["html"] == "An html description." assert resource["aiod_entry"]["date_modified"] == "2023-09-01T00:00:00+00:00" - assert resource["aiod_entry"]["status"] is None + assert resource["aiod_entry"]["status"] == "draft" global_fields = {"name", "description_plain", "description_html"} extra_fields = list(search_router.indexed_fields ^ global_fields) diff --git a/src/tests/testutils/default_instances.py b/src/tests/testutils/default_instances.py index 6cb8a9cc..b7906eda 100644 --- a/src/tests/testutils/default_instances.py +++ b/src/tests/testutils/default_instances.py @@ -13,7 +13,6 @@ from database.model.agent.contact import Contact from database.model.agent.organisation import Organisation from database.model.agent.person import Person -from database.model.concept.status import Status from database.model.dataset.dataset import Dataset from database.model.knowledge_asset.publication import Publication from database.model.models_and_experiments.experiment import Experiment @@ -24,11 +23,6 @@ from tests.testutils.paths import path_test_resources -@pytest.fixture -def draft() -> Status: - return Status(name="draft") - - @pytest.fixture(scope="session") def body_concept() -> dict: with open(path_test_resources() / "schemes" / "aiod" / "aiod_concept.json", "r") as f: diff --git a/src/tests/testutils/test_resource.py b/src/tests/testutils/test_resource.py index b8d59f48..e09e31d8 100644 --- a/src/tests/testutils/test_resource.py +++ b/src/tests/testutils/test_resource.py @@ -6,9 +6,8 @@ from sqlmodel import Field -from database.model.concept.aiod_entry import AIoDEntryORM +from database.model.concept.aiod_entry import AIoDEntryORM, EntryStatus from database.model.concept.concept import AIoDConcept, AIoDConceptBase -from database.model.concept.status import Status from routers.resource_router import ResourceRouter @@ -24,7 +23,7 @@ def factory( title=None, status=None, platform="example", platform_resource_identifier="1", date_deleted=None ): if status is None: - status = Status(name="draft") + status = EntryStatus.DRAFT return TestResource( title=title, platform=platform, diff --git a/src/uploaders/zenodo_uploader.py b/src/uploaders/zenodo_uploader.py index 72d0ea6d..fb28e379 100644 --- a/src/uploaders/zenodo_uploader.py +++ b/src/uploaders/zenodo_uploader.py @@ -13,7 +13,7 @@ from database.model.agent.contact import Contact from database.model.ai_asset.license import License from database.model.ai_resource.text import TextORM -from database.model.concept.status import Status +from database.model.concept.aiod_entry import EntryStatus from database.model.dataset.dataset import Dataset from database.model.platform.platform_names import PlatformName from database.session import DbSession @@ -58,7 +58,7 @@ def handle_upload( repo_id = platform_resource_id.split(":")[-1] zenodo_metadata = self._get_metadata_from_zenodo(repo_id, token) - if dataset.aiod_entry.status and dataset.aiod_entry.status.name == "published": + if dataset.aiod_entry.status and dataset.aiod_entry.status == "published": raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=( @@ -77,10 +77,7 @@ def handle_upload( record_url = new_zenodo_metadata["links"]["record"] distribution = self._get_distribution(repo_id, token, record_url) - new_status = session.query(Status).filter( - Status.name == "published" - ).first() or Status(name="published") - dataset.aiod_entry.status = new_status + dataset.aiod_entry.status = EntryStatus.PUBLISHED dataset.date_published = datetime.utcnow() else: distribution = self._get_distribution(repo_id, token) From 424c60044031c72a941d486312e0ffe0625e131a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 10 Dec 2024 12:13:28 +0200 Subject: [PATCH 2/5] Add entry status to the elastic search indices --- src/setup/logstash_setup/templates/sql_init.py | 3 ++- src/setup/logstash_setup/templates/sql_sync.py | 3 ++- src/tests/resources/elasticsearch/case_study_search.json | 1 + src/tests/resources/elasticsearch/dataset_search.json | 1 + .../resources/elasticsearch/educational_resource_search.json | 1 + src/tests/resources/elasticsearch/event_search.json | 1 + src/tests/resources/elasticsearch/experiment_search.json | 1 + src/tests/resources/elasticsearch/ml_model_search.json | 1 + src/tests/resources/elasticsearch/news_search.json | 1 + src/tests/resources/elasticsearch/organisation_search.json | 1 + src/tests/resources/elasticsearch/project_search.json | 1 + src/tests/resources/elasticsearch/publication_search.json | 1 + src/tests/resources/elasticsearch/service_search.json | 1 + src/tests/routers/search_routers/test_search_routers.py | 3 +-- 14 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/setup/logstash_setup/templates/sql_init.py b/src/setup/logstash_setup/templates/sql_init.py index 54d31411..868f25bf 100755 --- a/src/setup/logstash_setup/templates/sql_init.py +++ b/src/setup/logstash_setup/templates/sql_init.py @@ -4,7 +4,8 @@ {{entity_name}}.platform, text.plain as 'description_plain', text.html as 'description_html', - aiod_entry.date_modified{{extra_fields}}{{linked_fields}} + aiod_entry.date_modified, + aiod_entry.status{{extra_fields}}{{linked_fields}} FROM aiod.{{entity_name}} INNER JOIN aiod.aiod_entry ON aiod.{{entity_name}}.aiod_entry_identifier=aiod.aiod_entry.identifier LEFT JOIN aiod.text ON aiod.{{entity_name}}.description_identifier\ diff --git a/src/setup/logstash_setup/templates/sql_sync.py b/src/setup/logstash_setup/templates/sql_sync.py index 2ff419e5..2fcbfa7b 100755 --- a/src/setup/logstash_setup/templates/sql_sync.py +++ b/src/setup/logstash_setup/templates/sql_sync.py @@ -4,7 +4,8 @@ {{entity_name}}.platform, text.plain as 'description_plain', text.html as 'description_html', - aiod_entry.date_modified as 'date_modified'{{extra_fields}}{{linked_fields}} + aiod_entry.date_modified as 'date_modified', + aiod_entry.status{{extra_fields}}{{linked_fields}} FROM aiod.{{entity_name}} INNER JOIN aiod.aiod_entry ON aiod.{{entity_name}}.aiod_entry_identifier=aiod.aiod_entry.identifier LEFT JOIN aiod.text ON aiod.{{entity_name}}.description_identifier\ diff --git a/src/tests/resources/elasticsearch/case_study_search.json b/src/tests/resources/elasticsearch/case_study_search.json index 4c16c575..e22e0d0c 100644 --- a/src/tests/resources/elasticsearch/case_study_search.json +++ b/src/tests/resources/elasticsearch/case_study_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/dataset_search.json b/src/tests/resources/elasticsearch/dataset_search.json index e9dfca5d..d0e9c023 100644 --- a/src/tests/resources/elasticsearch/dataset_search.json +++ b/src/tests/resources/elasticsearch/dataset_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/educational_resource_search.json b/src/tests/resources/elasticsearch/educational_resource_search.json index 8e6cbf8e..3ddf7ce1 100644 --- a/src/tests/resources/elasticsearch/educational_resource_search.json +++ b/src/tests/resources/elasticsearch/educational_resource_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/event_search.json b/src/tests/resources/elasticsearch/event_search.json index da590288..cfdc9e85 100644 --- a/src/tests/resources/elasticsearch/event_search.json +++ b/src/tests/resources/elasticsearch/event_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/experiment_search.json b/src/tests/resources/elasticsearch/experiment_search.json index 8cbbe4b7..b25de89c 100644 --- a/src/tests/resources/elasticsearch/experiment_search.json +++ b/src/tests/resources/elasticsearch/experiment_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/ml_model_search.json b/src/tests/resources/elasticsearch/ml_model_search.json index bd970972..bcc0d191 100644 --- a/src/tests/resources/elasticsearch/ml_model_search.json +++ b/src/tests/resources/elasticsearch/ml_model_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/news_search.json b/src/tests/resources/elasticsearch/news_search.json index d719a826..45699d2f 100644 --- a/src/tests/resources/elasticsearch/news_search.json +++ b/src/tests/resources/elasticsearch/news_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/organisation_search.json b/src/tests/resources/elasticsearch/organisation_search.json index e7dc2877..2824b548 100644 --- a/src/tests/resources/elasticsearch/organisation_search.json +++ b/src/tests/resources/elasticsearch/organisation_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/project_search.json b/src/tests/resources/elasticsearch/project_search.json index 5c1aa78a..237617ad 100644 --- a/src/tests/resources/elasticsearch/project_search.json +++ b/src/tests/resources/elasticsearch/project_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/publication_search.json b/src/tests/resources/elasticsearch/publication_search.json index 8e8e3d03..9ce99127 100644 --- a/src/tests/resources/elasticsearch/publication_search.json +++ b/src/tests/resources/elasticsearch/publication_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/service_search.json b/src/tests/resources/elasticsearch/service_search.json index aca61fe6..ddf6df93 100644 --- a/src/tests/resources/elasticsearch/service_search.json +++ b/src/tests/resources/elasticsearch/service_search.json @@ -21,6 +21,7 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", + "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/routers/search_routers/test_search_routers.py b/src/tests/routers/search_routers/test_search_routers.py index 14a9557e..a64e730a 100644 --- a/src/tests/routers/search_routers/test_search_routers.py +++ b/src/tests/routers/search_routers/test_search_routers.py @@ -10,8 +10,7 @@ from tests.testutils.paths import path_test_resources -# @pytest.mark.parametrize("search_router", sr.router_list) -@pytest.mark.skip("Separate out ES updates for next commit") +@pytest.mark.parametrize("search_router", sr.router_list) def test_search_happy_path(client: TestClient, search_router): mock_elasticsearch(filename_mock=f"{search_router.es_index}_search.json") From a86bc254944c6fdb19d10782c43139f2775ea78a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 17 Dec 2024 10:59:25 +0200 Subject: [PATCH 3/5] Revert "Add entry status to the elastic search indices" This reverts commit 6038b0c0e8c359a1a5c8275e86fa60dbd958e4b4. --- src/setup/logstash_setup/templates/sql_init.py | 3 +-- src/setup/logstash_setup/templates/sql_sync.py | 3 +-- src/tests/resources/elasticsearch/case_study_search.json | 1 - src/tests/resources/elasticsearch/dataset_search.json | 1 - .../resources/elasticsearch/educational_resource_search.json | 1 - src/tests/resources/elasticsearch/event_search.json | 1 - src/tests/resources/elasticsearch/experiment_search.json | 1 - src/tests/resources/elasticsearch/ml_model_search.json | 1 - src/tests/resources/elasticsearch/news_search.json | 1 - src/tests/resources/elasticsearch/organisation_search.json | 1 - src/tests/resources/elasticsearch/project_search.json | 1 - src/tests/resources/elasticsearch/publication_search.json | 1 - src/tests/resources/elasticsearch/service_search.json | 1 - src/tests/routers/search_routers/test_search_routers.py | 3 ++- 14 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/setup/logstash_setup/templates/sql_init.py b/src/setup/logstash_setup/templates/sql_init.py index 868f25bf..54d31411 100755 --- a/src/setup/logstash_setup/templates/sql_init.py +++ b/src/setup/logstash_setup/templates/sql_init.py @@ -4,8 +4,7 @@ {{entity_name}}.platform, text.plain as 'description_plain', text.html as 'description_html', - aiod_entry.date_modified, - aiod_entry.status{{extra_fields}}{{linked_fields}} + aiod_entry.date_modified{{extra_fields}}{{linked_fields}} FROM aiod.{{entity_name}} INNER JOIN aiod.aiod_entry ON aiod.{{entity_name}}.aiod_entry_identifier=aiod.aiod_entry.identifier LEFT JOIN aiod.text ON aiod.{{entity_name}}.description_identifier\ diff --git a/src/setup/logstash_setup/templates/sql_sync.py b/src/setup/logstash_setup/templates/sql_sync.py index 2fcbfa7b..2ff419e5 100755 --- a/src/setup/logstash_setup/templates/sql_sync.py +++ b/src/setup/logstash_setup/templates/sql_sync.py @@ -4,8 +4,7 @@ {{entity_name}}.platform, text.plain as 'description_plain', text.html as 'description_html', - aiod_entry.date_modified as 'date_modified', - aiod_entry.status{{extra_fields}}{{linked_fields}} + aiod_entry.date_modified as 'date_modified'{{extra_fields}}{{linked_fields}} FROM aiod.{{entity_name}} INNER JOIN aiod.aiod_entry ON aiod.{{entity_name}}.aiod_entry_identifier=aiod.aiod_entry.identifier LEFT JOIN aiod.text ON aiod.{{entity_name}}.description_identifier\ diff --git a/src/tests/resources/elasticsearch/case_study_search.json b/src/tests/resources/elasticsearch/case_study_search.json index e22e0d0c..4c16c575 100644 --- a/src/tests/resources/elasticsearch/case_study_search.json +++ b/src/tests/resources/elasticsearch/case_study_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/dataset_search.json b/src/tests/resources/elasticsearch/dataset_search.json index d0e9c023..e9dfca5d 100644 --- a/src/tests/resources/elasticsearch/dataset_search.json +++ b/src/tests/resources/elasticsearch/dataset_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/educational_resource_search.json b/src/tests/resources/elasticsearch/educational_resource_search.json index 3ddf7ce1..8e6cbf8e 100644 --- a/src/tests/resources/elasticsearch/educational_resource_search.json +++ b/src/tests/resources/elasticsearch/educational_resource_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/event_search.json b/src/tests/resources/elasticsearch/event_search.json index cfdc9e85..da590288 100644 --- a/src/tests/resources/elasticsearch/event_search.json +++ b/src/tests/resources/elasticsearch/event_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/experiment_search.json b/src/tests/resources/elasticsearch/experiment_search.json index b25de89c..8cbbe4b7 100644 --- a/src/tests/resources/elasticsearch/experiment_search.json +++ b/src/tests/resources/elasticsearch/experiment_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/ml_model_search.json b/src/tests/resources/elasticsearch/ml_model_search.json index bcc0d191..bd970972 100644 --- a/src/tests/resources/elasticsearch/ml_model_search.json +++ b/src/tests/resources/elasticsearch/ml_model_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/news_search.json b/src/tests/resources/elasticsearch/news_search.json index 45699d2f..d719a826 100644 --- a/src/tests/resources/elasticsearch/news_search.json +++ b/src/tests/resources/elasticsearch/news_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/organisation_search.json b/src/tests/resources/elasticsearch/organisation_search.json index 2824b548..e7dc2877 100644 --- a/src/tests/resources/elasticsearch/organisation_search.json +++ b/src/tests/resources/elasticsearch/organisation_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/project_search.json b/src/tests/resources/elasticsearch/project_search.json index 237617ad..5c1aa78a 100644 --- a/src/tests/resources/elasticsearch/project_search.json +++ b/src/tests/resources/elasticsearch/project_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/publication_search.json b/src/tests/resources/elasticsearch/publication_search.json index 9ce99127..8e8e3d03 100644 --- a/src/tests/resources/elasticsearch/publication_search.json +++ b/src/tests/resources/elasticsearch/publication_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/resources/elasticsearch/service_search.json b/src/tests/resources/elasticsearch/service_search.json index ddf6df93..aca61fe6 100644 --- a/src/tests/resources/elasticsearch/service_search.json +++ b/src/tests/resources/elasticsearch/service_search.json @@ -21,7 +21,6 @@ "_source" : { "identifier" : 1, "date_modified" : "2023-09-01T00:00:00.000Z", - "status": "draft", "name" : "A name.", "platform" : "A platform.", "description_plain" : "A plain text description.", diff --git a/src/tests/routers/search_routers/test_search_routers.py b/src/tests/routers/search_routers/test_search_routers.py index a64e730a..14a9557e 100644 --- a/src/tests/routers/search_routers/test_search_routers.py +++ b/src/tests/routers/search_routers/test_search_routers.py @@ -10,7 +10,8 @@ from tests.testutils.paths import path_test_resources -@pytest.mark.parametrize("search_router", sr.router_list) +# @pytest.mark.parametrize("search_router", sr.router_list) +@pytest.mark.skip("Separate out ES updates for next commit") def test_search_happy_path(client: TestClient, search_router): mock_elasticsearch(filename_mock=f"{search_router.es_index}_search.json") From cc1ceae426b85d3c1970dea9e0bfbc40fba7f49c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 17 Dec 2024 16:56:41 +0200 Subject: [PATCH 4/5] Add convenience script to allow setting alembic revision --- scripts/set_alembic.sh | 1 + 1 file changed, 1 insertion(+) create mode 100755 scripts/set_alembic.sh diff --git a/scripts/set_alembic.sh b/scripts/set_alembic.sh new file mode 100755 index 00000000..2c88a3c2 --- /dev/null +++ b/scripts/set_alembic.sh @@ -0,0 +1 @@ +docker exec -it sqlserver mysql -uroot -pok --database=aiod -e "UPDATE alembic_version SET version_num = '$1'" \ No newline at end of file From e261d2b522e1b2b93793871b11d9009b40ade209 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 17 Dec 2024 16:57:49 +0200 Subject: [PATCH 5/5] Add migration script to make Entry Status an Enum --- .../1662d64ebe23_make_draft_status_enum.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 alembic/alembic/versions/1662d64ebe23_make_draft_status_enum.py diff --git a/alembic/alembic/versions/1662d64ebe23_make_draft_status_enum.py b/alembic/alembic/versions/1662d64ebe23_make_draft_status_enum.py new file mode 100644 index 00000000..db6653fa --- /dev/null +++ b/alembic/alembic/versions/1662d64ebe23_make_draft_status_enum.py @@ -0,0 +1,90 @@ +"""make draft status enum + +Revision ID: 1662d64ebe23 +Revises: d09ed8ad4533 +Create Date: 2024-12-17 09:02:30.480835 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import Column, INT, String, Enum + +from database.model.field_length import NORMAL +from database.model.concept.aiod_entry import EntryStatus + +# revision identifiers, used by Alembic. +revision: str = "1662d64ebe23" +down_revision: Union[str, None] = "d09ed8ad4533" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table("aiod_entry_status_link") + op.add_column( + "aiod_entry", + Column("status", Enum(EntryStatus)), + ) + op.execute( + """ + UPDATE aiod_entry + INNER JOIN status + ON status.identifier = aiod_entry.status_identifier + SET aiod_entry.status = status.name + """ + ) + op.drop_constraint( + constraint_name="aiod_entry_ibfk_1", + table_name="aiod_entry", + type_="foreignkey", + ) + op.drop_column( + table_name="aiod_entry", + column_name="status_identifier", + ) + op.drop_table("status") + + +def downgrade() -> None: + # No need to recreate table status link, it was not used. + op.create_table( + "status", + Column("identifier", type_=INT, primary_key=True), + Column( + "name", + unique=True, + type_=String(NORMAL), + index=True, + ), + ) + op.execute( + """ + INSERT INTO status + VALUES (1, 'draft'), (2, 'published'), (3, 'rejected'), (4, 'submitted') + """ + ) + op.add_column( + "aiod_entry", + Column("status_identifier", INT), + ) + op.execute( + """ + UPDATE aiod_entry + INNER JOIN status + ON aiod_entry.status = status.name + SET aiod_entry.status_identifier = status.identifier + """ + ) + op.drop_column( + "aiod_entry", + "status", + ) + op.create_foreign_key( + "aiod_entry_ibfk_1", + "aiod_entry", + "status", + ["status_identifier"], + ["identifier"], + )