Merge pull request #44 from meaningfy-ws/feature/TED-160/178

Feature/ted 160/178

ted_sws/__init__.py

@@ -12,8 +12,8 @@
 import os
 
 import dotenv
-from ted_sws.adapters.config_resolver import VaultAndEnvConfigResolver
-from ted_sws.adapters.vault_secrets_store import VaultSecretsStore
+from ted_sws.core.adapters.config_resolver import VaultAndEnvConfigResolver
+from ted_sws.core.adapters.vault_secrets_store import VaultSecretsStore
 
 dotenv.load_dotenv(verbose=True, override=True)
 

ted_sws/adapters/config_resolver.py → ted_sws/core/adapters/config_resolver.py

@@ -13,7 +13,7 @@
 import os
 from abc import ABC
 
-from ted_sws.adapters.vault_secrets_store import VaultSecretsStore
+from ted_sws.core.adapters.vault_secrets_store import VaultSecretsStore
 
 logger = logging.getLogger(__name__)
 

ted_sws/domain/model/manifestation.py → ted_sws/core/model/manifestation.py

@@ -10,7 +10,7 @@
 
 from pydantic import BaseModel, Field
 
-from ted_sws.domain.model import PropertyBaseModel
+from ted_sws.core.model import PropertyBaseModel
 
 
 class ManifestationMimeType(Enum):

ted_sws/domain/model/metadata.py → ted_sws/core/model/metadata.py

@@ -11,7 +11,7 @@
 
 from pydantic.annotated_types import NamedTuple
 
-from ted_sws.domain.model import PropertyBaseModel
+from ted_sws.core.model import PropertyBaseModel
 
 
 class Metadata(PropertyBaseModel):

ted_sws/domain/model/notice.py → ted_sws/core/model/notice.py

@@ -20,10 +20,10 @@
 
 from pydantic import Field
 
-from ted_sws.domain.model import PropertyBaseModel
-from ted_sws.domain.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
+from ted_sws.core.model import PropertyBaseModel
+from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
     RDFValidationManifestation
-from ted_sws.domain.model.metadata import TEDMetadata, NormalisedMetadata
+from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata
 
 
 class UnsupportedStatusTransition(Exception):
@@ -312,6 +312,7 @@ def update_status_to(self, new_status: NoticeStatus):
                 raise UnsupportedStatusTransition(
                     f"Unsupported transition from state {self._status} to state {new_status}.")
         elif self._status > new_status:
+            # TODO: implement delete actions
             self._status = new_status
             if new_status < NoticeStatus.NORMALISED_METADATA:
                 self._normalised_metadata = None

ted_sws/domain/model/transform.py → ted_sws/core/model/transform.py

@@ -11,7 +11,7 @@
 from datetime import datetime
 from typing import List
 
-from ted_sws.domain.model import PropertyBaseModel
+from ted_sws.core.model import PropertyBaseModel
 
 
 class MappingSuiteComponent(PropertyBaseModel, abc.ABC):

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -6,8 +6,8 @@
 from pymongo import MongoClient
 
 from ted_sws import config
-from ted_sws.domain.adapters.repository_abc import MappingSuiteRepositoryABC
-from ted_sws.domain.model.transform import MappingSuite, FileResource, TransformationRuleSet, SHACLTestSuite, \
+from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC
+from ted_sws.core.model.transform import MappingSuite, FileResource, TransformationRuleSet, SHACLTestSuite, \
     SPARQLTestSuite, MetadataConstraints, TransformationTestData
 
 METADATA_FILE_NAME = "metadata.json"
@@ -28,13 +28,14 @@ class MappingSuiteRepositoryMongoDB(MappingSuiteRepositoryABC):
     _collection_name = "mapping_suite_collection"
     _database_name = config.MONGO_DB_AGGREGATES_DATABASE_NAME
 
-    def __init__(self, mongodb_client: MongoClient):
+    def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name):
         """
 
         :param mongodb_client:
         :param database_name:
         """
         mongodb_client = mongodb_client
+        self._database_name = database_name
         notice_db = mongodb_client[self._database_name]
         self.collection = notice_db[self._collection_name]
 
@@ -224,7 +225,6 @@ def _write_package_validation_rules(self, mapping_suite: MappingSuite):
                                        path=sparql_test_suite_path
                                        )
 
-
     def _write_test_data_package(self, mapping_suite: MappingSuite):
         """
             This method writes the test data to a dedicated folder in the package.
@@ -238,7 +238,7 @@ def _write_test_data_package(self, mapping_suite: MappingSuite):
                                    path=test_data_path
                                    )
 
-    def _read_test_data_package(self, package_path: pathlib.Path)-> TransformationTestData:
+    def _read_test_data_package(self, package_path: pathlib.Path) -> TransformationTestData:
         """
             This method reads the test data from the package.
         :param package_path:
@@ -248,7 +248,6 @@ def _read_test_data_package(self, package_path: pathlib.Path)-> TransformationTe
         test_data = self._read_file_resources(path=test_data_path)
         return TransformationTestData(test_data=test_data)
 
-
     def _write_mapping_suite_package(self, mapping_suite: MappingSuite):
         """
             This method creates a package based on data from mapping_suite.

ted_sws/data_manager/adapters/notice_repository.py

@@ -3,8 +3,8 @@
 from pymongo import MongoClient
 
 from ted_sws import config
-from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
-from ted_sws.domain.model.notice import Notice, NoticeStatus
+from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC
+from ted_sws.core.model.notice import Notice, NoticeStatus
 
 logger = logging.getLogger(__name__)
 
@@ -17,8 +17,9 @@ class NoticeRepository(NoticeRepositoryABC):
     _collection_name = "notice_collection"
     _database_name = config.MONGO_DB_AGGREGATES_DATABASE_NAME
 
-    def __init__(self, mongodb_client: MongoClient):
+    def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name):
         mongodb_client = mongodb_client
+        self._database_name = database_name
         notice_db = mongodb_client[self._database_name]
         self.collection = notice_db[self._collection_name]
 

ted_sws/domain/adapters/repository_abc.py → ted_sws/data_manager/adapters/repository_abc.py

@@ -1,8 +1,8 @@
 import abc
 from typing import Iterator
 
-from ted_sws.domain.model.notice import Notice
-from ted_sws.domain.model.transform import MappingSuite
+from ted_sws.core.model.notice import Notice
+from ted_sws.core.model.transform import MappingSuite
 
 
 class RepositoryABC(abc.ABC):

ted_sws/metadata_normaliser/entrypoints/generate_mapping_resources.py

@@ -2,7 +2,7 @@
 import logging
 import pathlib
 
-from ted_sws.adapters.sparql_triple_store import SPARQLTripleStore, TripleStoreABC
+from ted_sws.core.adapters.sparql_triple_store import SPARQLTripleStore, TripleStoreABC
 from ted_sws.metadata_normaliser.resources import MAPPING_FILES_PATH, QUERIES_PATH
 
 logger = logging.getLogger(__name__)

ted_sws/metadata_normaliser/model/metadata.py

@@ -1,7 +1,7 @@
 import datetime
 from typing import List
 
-from ted_sws.domain.model.metadata import Metadata, CompositeTitle, LanguageTaggedString, EncodedValue
+from ted_sws.core.model.metadata import Metadata, CompositeTitle, LanguageTaggedString, EncodedValue
 
 
 class ExtractedMetadata(Metadata):

ted_sws/metadata_normaliser/services/metadata_normalizer.py

@@ -4,8 +4,8 @@
 import pandas as pd
 
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepositoryABC
-from ted_sws.domain.model.metadata import NormalisedMetadata, LanguageTaggedString
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString
+from ted_sws.core.model.notice import Notice
 from ted_sws.metadata_normaliser.model.metadata import ExtractedMetadata
 from ted_sws.metadata_normaliser.services.xml_manifestation_metadata_extractor import XMLManifestationMetadataExtractor
 from ted_sws.metadata_normaliser.resources.mapping_files_registry import MappingFilesRegistry

ted_sws/metadata_normaliser/services/xml_manifestation_metadata_extractor.py

@@ -1,7 +1,7 @@
 import xml.etree.ElementTree as ET
 from io import StringIO
 
-from ted_sws.domain.model.manifestation import XMLManifestation
+from ted_sws.core.model.manifestation import XMLManifestation
 from ted_sws.metadata_normaliser.model.metadata import ExtractedMetadata, LanguageTaggedString, CompositeTitle, \
     EncodedValue
 from ted_sws.metadata_normaliser.services.xpath_registry import XpathRegistry

ted_sws/notice_fetcher/services/notice_fetcher.py

@@ -1,10 +1,10 @@
 import abc
 from datetime import date
 
-from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
-from ted_sws.domain.model.manifestation import XMLManifestation
-from ted_sws.domain.model.metadata import TEDMetadata
-from ted_sws.domain.model.notice import Notice
+from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC
+from ted_sws.core.model.manifestation import XMLManifestation
+from ted_sws.core.model.metadata import TEDMetadata
+from ted_sws.core.model.notice import Notice
 from ted_sws.notice_fetcher.adapters.ted_api_abc import TedAPIAdapterABC
 
 

ted_sws/notice_packager/model/metadata.py

@@ -14,7 +14,7 @@
 
 from pydantic import validator
 
-from ted_sws.domain.model.metadata import Metadata
+from ted_sws.core.model.metadata import Metadata
 
 WORK_AGENT = "PUBL"
 PUBLICATION_FREQUENCY = "OTHER"

ted_sws/notice_packager/services/notice_packager.py

@@ -15,7 +15,7 @@
 from tempfile import TemporaryDirectory
 from typing import List, Union
 
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.notice import Notice
 from ted_sws.metadata_normaliser.model.metadata import ExtractedMetadata
 from ted_sws.metadata_normaliser.services.xml_manifestation_metadata_extractor import XMLManifestationMetadataExtractor
 from ted_sws.notice_packager.adapters.archiver import ArchiverFactory, ARCHIVE_ZIP_FORMAT, PATH_TYPE, \

ted_sws/notice_transformer/services/notice_transformer.py

@@ -4,9 +4,9 @@
 
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem, \
     TRANSFORM_PACKAGE_NAME, RESOURCES_PACKAGE_NAME
-from ted_sws.domain.model.manifestation import RDFManifestation, XMLManifestation
-from ted_sws.domain.model.notice import Notice, NoticeStatus
-from ted_sws.domain.model.transform import MappingSuite, FileResource
+from ted_sws.core.model.manifestation import RDFManifestation, XMLManifestation
+from ted_sws.core.model.notice import Notice, NoticeStatus
+from ted_sws.core.model.transform import MappingSuite, FileResource
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC
 
 DATA_SOURCE_PACKAGE = "data"

tests/conftest.py

@@ -4,9 +4,9 @@
 
 import pytest
 
-from ted_sws.domain.model.manifestation import XMLManifestation
-from ted_sws.domain.model.metadata import TEDMetadata, LanguageTaggedString
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.manifestation import XMLManifestation
+from ted_sws.core.model.metadata import TEDMetadata, LanguageTaggedString
+from ted_sws.core.model.notice import Notice
 from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
 from tests import TEST_DATA_PATH

tests/e2e/data_manager/test_notice_repository.py

@@ -1,17 +1,17 @@
 import pytest
 
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-from ted_sws.domain.model.manifestation import XMLManifestation
-from ted_sws.domain.model.metadata import TEDMetadata
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.manifestation import XMLManifestation
+from ted_sws.core.model.metadata import TEDMetadata
+from ted_sws.core.model.notice import Notice
 
 NOTICE_TED_ID = "123456"
-TEST_DB_NAME = 'test_db'
+TEST_DATABASE_NAME = "test_database_name"
 
 
 def test_notice_repository_create(mongodb_client):
-    mongodb_client.drop_database(TEST_DB_NAME)
-    notice_repository = NoticeRepository(mongodb_client=mongodb_client, database_name=TEST_DB_NAME)
+    mongodb_client.drop_database(TEST_DATABASE_NAME)
+    notice_repository = NoticeRepository(mongodb_client=mongodb_client, database_name=TEST_DATABASE_NAME)
     notice = Notice(ted_id=NOTICE_TED_ID, original_metadata=TEDMetadata(**{"AA": "Metadata"}),
                     xml_manifestation=XMLManifestation(object_data="HELLO"))
     notice_repository.add(notice)
@@ -29,4 +29,4 @@ def test_notice_repository_create(mongodb_client):
     assert result_notice
     assert result_notice.ted_id == NOTICE_TED_ID
     assert result_notice.original_metadata.AA == "Updated metadata"
-    mongodb_client.drop_database(TEST_DB_NAME)
+    mongodb_client.drop_database(TEST_DATABASE_NAME)

tests/e2e/metadata_normaliser/test_sparql_triple_store.py

@@ -1,9 +1,7 @@
-import pathlib
-
 import pandas as pd
 import pytest
 
-from ted_sws.adapters.sparql_triple_store import SPARQLTripleStore
+from ted_sws.core.adapters.sparql_triple_store import SPARQLTripleStore
 from tests import TEST_DATA_PATH
 
 

tests/e2e/notice_fetcher/test_notice_fetcher.py

@@ -1,5 +1,5 @@
 import datetime
-from ted_sws.domain.model.notice import Notice, NoticeStatus
+from ted_sws.core.model.notice import Notice, NoticeStatus
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
 
 

tests/e2e/notice_transformer/test_rml_mapper.py

@@ -1,8 +1,9 @@
 from ted_sws import config
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper
+from tests.test_data.notice_transformer import RML_MAPPER_TEST_RDF_RESULT
 
 
 def test_rml_mapper(rml_test_package_path):
     rml_mapper = RMLMapper(rml_mapper_path=config.RML_MAPPER_PATH)
     rdf_result = rml_mapper.execute(package_path=rml_test_package_path)
-    assert rdf_result
+    assert rdf_result == RML_MAPPER_TEST_RDF_RESULT

tests/fakes/fake_notice.py

@@ -1,5 +1,5 @@
-from ted_sws.domain.model.notice import Notice
-from ted_sws.domain.model.manifestation import XMLManifestation
+from ted_sws.core.model.notice import Notice
+from ted_sws.core.model.manifestation import XMLManifestation
 
 
 class FakeNotice(Notice):

tests/fakes/fake_repository.py

@@ -1,7 +1,7 @@
 from typing import Iterator
 
-from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
-from ted_sws.domain.model.notice import Notice, NoticeStatus
+from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC
+from ted_sws.core.model.notice import Notice, NoticeStatus
 
 
 class FakeNoticeRepository(NoticeRepositoryABC):

tests/fakes/fake_triple_store.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from ted_sws.adapters.sparql_triple_store import TripleStoreABC
+from ted_sws.core.adapters.sparql_triple_store import TripleStoreABC
 
 
 class FakeTripleStore(TripleStoreABC):

tests/features/model/conftest.py

@@ -9,10 +9,10 @@
 
 import pytest
 
-from ted_sws.domain.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, \
+from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, \
     RDFValidationManifestation
-from ted_sws.domain.model.metadata import TEDMetadata, NormalisedMetadata
-from ted_sws.domain.model.notice import Notice, NoticeStatus
+from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata
+from ted_sws.core.model.notice import Notice, NoticeStatus
 
 
 @pytest.fixture

tests/features/model/test_creating_notice.py

@@ -1,8 +1,8 @@
 from pytest_bdd import scenario, given, when, then
 
-from ted_sws.domain.model.manifestation import XMLManifestation
-from ted_sws.domain.model.metadata import TEDMetadata
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.manifestation import XMLManifestation
+from ted_sws.core.model.metadata import TEDMetadata
+from ted_sws.core.model.notice import Notice
 
 
 @scenario("test_creating_notice.feature", "Create a bare minimum notice")

tests/features/model/test_notice_operations.py

@@ -1,9 +1,9 @@
 import pytest
 from pytest_bdd import scenario, given, when, then, parsers
 
-from ted_sws.domain.model.manifestation import RDFManifestation, RDFValidationManifestation, METSManifestation
-from ted_sws.domain.model.metadata import NormalisedMetadata
-from ted_sws.domain.model.notice import NoticeStatus
+from ted_sws.core.model.manifestation import RDFManifestation, RDFValidationManifestation, METSManifestation
+from ted_sws.core.model.metadata import NormalisedMetadata
+from ted_sws.core.model.notice import NoticeStatus
 from tests.features import str2bool
 
 

tests/features/notice_fetcher/test_fetching_types.py

@@ -1,7 +1,7 @@
 import pytest
 from pytest_bdd import scenario, given, when, then
 
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.notice import Notice
 from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
 

tests/features/notice_fetcher/test_notice_fetcher.py

@@ -2,7 +2,7 @@
 from _pytest._code import ExceptionInfo
 from pytest_bdd import scenario, given, when, then
 
-from ted_sws.domain.model.notice import Notice
+from ted_sws.core.model.notice import Notice
 from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher