diff --git a/dags/operators/DagBatchPipelineOperator.py b/dags/operators/DagBatchPipelineOperator.py index 4480187ac..b30416329 100644 --- a/dags/operators/DagBatchPipelineOperator.py +++ b/dags/operators/DagBatchPipelineOperator.py @@ -16,7 +16,7 @@ NOTICE_IDS_KEY = "notice_ids" START_WITH_STEP_NAME_KEY = "start_with_step_name" EXECUTE_ONLY_ONE_STEP_KEY = "execute_only_one_step" -DEFAULT_NUBER_OF_CELERY_WORKERS = 144 +DEFAULT_NUBER_OF_CELERY_WORKERS = 144 #TODO: revise this config NOTICE_PROCESS_WORKFLOW_DAG_NAME = "notice_process_workflow" DEFAULT_START_WITH_TASK_ID = "notice_normalisation_pipeline" DEFAULT_PIPELINE_NAME_FOR_LOGS = "unknown_pipeline_name" diff --git a/dags/pipelines/notice_processor_pipelines.py b/dags/pipelines/notice_processor_pipelines.py index 23d7d5104..e0489664c 100644 --- a/dags/pipelines/notice_processor_pipelines.py +++ b/dags/pipelines/notice_processor_pipelines.py @@ -10,7 +10,7 @@ def notice_normalisation_pipeline(notice: Notice, mongodb_client: MongoClient) - """ from ted_sws.data_sampler.services.notice_xml_indexer import index_notice from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice - + notice.update_status_to(new_status=NoticeStatus.RAW) indexed_notice = index_notice(notice=notice) normalised_notice = normalise_notice(notice=indexed_notice) @@ -27,7 +27,7 @@ def notice_transformation_pipeline(notice: Notice, mongodb_client: MongoClient) from ted_sws.notice_transformer.services.notice_transformer import transform_notice from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB - + notice.update_status_to(new_status=NoticeStatus.NORMALISED_METADATA) mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) result = notice_eligibility_checker(notice=notice, mapping_suite_repository=mapping_suite_repository) if not result: @@ -57,7 +57,7 @@ def notice_validation_pipeline(notice: Notice, mongodb_client: MongoClient) -> N from ted_sws.notice_validator.services.xpath_coverage_runner import validate_xpath_coverage_notice from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB from ted_sws.event_manager.services.log import log_notice_info - + notice.update_status_to(new_status=NoticeStatus.DISTILLED) mapping_suite_id = notice.distilled_rdf_manifestation.mapping_suite_id mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) mapping_suite = mapping_suite_repository.get(reference=mapping_suite_id) @@ -82,6 +82,7 @@ def notice_package_pipeline(notice: Notice, mongodb_client: MongoClient) -> Noti """ from ted_sws.notice_packager.services.notice_packager import package_notice + notice.update_status_to(new_status=NoticeStatus.VALIDATED) # TODO: Implement notice package eligiblity notice.set_is_eligible_for_packaging(eligibility=True) packaged_notice = package_notice(notice=notice) @@ -95,7 +96,7 @@ def notice_publish_pipeline(notice: Notice, mongodb_client: MongoClient) -> Noti from ted_sws.notice_publisher.services.notice_publisher import publish_notice, publish_notice_rdf_into_s3 from ted_sws.event_manager.services.log import log_notice_error from ted_sws import config - + notice.update_status_to(new_status=NoticeStatus.PACKAGED) if config.S3_PUBLISH_ENABLED: published_into_s3 = publish_notice_rdf_into_s3(notice=notice) if not published_into_s3: @@ -106,4 +107,5 @@ def notice_publish_pipeline(notice: Notice, mongodb_client: MongoClient) -> Noti if result: return NoticePipelineOutput(notice=notice) else: + notice.set_is_eligible_for_publishing(eligibility=False) return NoticePipelineOutput(notice=notice, processed=False) diff --git a/dags/pipelines/notice_selectors_pipelines.py b/dags/pipelines/notice_selectors_pipelines.py index 206cda21c..541ae6035 100644 --- a/dags/pipelines/notice_selectors_pipelines.py +++ b/dags/pipelines/notice_selectors_pipelines.py @@ -7,11 +7,20 @@ PUBLICATION_DATE = "normalised_metadata.publication_date" -def build_selector_mongodb_filter(notice_status: str, form_number: str = None, +def build_selector_mongodb_filter(notice_statuses: List[str], form_number: str = None, start_date: str = None, end_date: str = None, xsd_version: str = None) -> dict: + """ + + :param notice_statuses: + :param form_number: + :param start_date: + :param end_date: + :param xsd_version: + :return: + """ from datetime import datetime - mongodb_filter = {NOTICE_STATUS: notice_status} + mongodb_filter = {NOTICE_STATUS: {"$in": notice_statuses}} if form_number: mongodb_filter[FORM_NUMBER] = form_number if start_date and end_date: @@ -26,21 +35,26 @@ def build_selector_mongodb_filter(notice_status: str, form_number: str = None, def notice_ids_selector_by_status(notice_statuses: List[NoticeStatus], form_number: str = None, start_date: str = None, end_date: str = None, xsd_version: str = None) -> List[str]: + """ + + :param notice_statuses: + :param form_number: + :param start_date: + :param end_date: + :param xsd_version: + :return: + """ from pymongo import MongoClient from ted_sws import config from ted_sws.data_manager.adapters.notice_repository import NoticeRepository, NOTICE_TED_ID mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL) notice_repository = NoticeRepository(mongodb_client=mongodb_client) - notice_ids = [] - for notice_status in notice_statuses: - mongodb_filter = build_selector_mongodb_filter(notice_status=str(notice_status), - form_number=form_number, - start_date=start_date, - end_date=end_date, - xsd_version=xsd_version - ) - mongodb_result_iterator = notice_repository.collection.find(mongodb_filter, {NOTICE_TED_ID: 1}) - notice_ids.extend([result_dict[NOTICE_TED_ID] for result_dict in mongodb_result_iterator]) - - return notice_ids + notice_statuses = [str(notice_status) for notice_status in notice_statuses] + mongodb_filter = build_selector_mongodb_filter(notice_statuses=notice_statuses, + form_number=form_number, + start_date=start_date, + end_date=end_date, + xsd_version=xsd_version) + mongodb_result_iterator = notice_repository.collection.find(mongodb_filter, {NOTICE_TED_ID: 1}) + return [result_dict[NOTICE_TED_ID] for result_dict in mongodb_result_iterator] diff --git a/dags/selector_repackage_process_orchestrator.py b/dags/selector_repackage_process_orchestrator.py index 5ac2d2dc6..5a476d547 100644 --- a/dags/selector_repackage_process_orchestrator.py +++ b/dags/selector_repackage_process_orchestrator.py @@ -13,6 +13,7 @@ DAG_NAME = "selector_re_package_process_orchestrator" RE_PACKAGE_TARGET_NOTICE_STATES = [NoticeStatus.VALIDATED, NoticeStatus.INELIGIBLE_FOR_PACKAGING, + NoticeStatus.ELIGIBLE_FOR_PACKAGING, NoticeStatus.INELIGIBLE_FOR_PUBLISHING] TRIGGER_NOTICE_PROCESS_WORKFLOW_TASK_ID = "trigger_notice_process_workflow" FORM_NUMBER_DAG_PARAM = "form_number" diff --git a/dags/selector_republish_process_orchestrator.py b/dags/selector_republish_process_orchestrator.py index 76cc208fc..b91b231b0 100644 --- a/dags/selector_republish_process_orchestrator.py +++ b/dags/selector_republish_process_orchestrator.py @@ -13,7 +13,9 @@ DAG_NAME = "selector_re_publish_process_orchestrator" -RE_PUBLISH_TARGET_NOTICE_STATES = [NoticeStatus.PUBLICLY_UNAVAILABLE, NoticeStatus.ELIGIBLE_FOR_PUBLISHING] +RE_PUBLISH_TARGET_NOTICE_STATES = [NoticeStatus.ELIGIBLE_FOR_PUBLISHING, NoticeStatus.INELIGIBLE_FOR_PUBLISHING, + NoticeStatus.PACKAGED + ] TRIGGER_NOTICE_PROCESS_WORKFLOW_TASK_ID = "trigger_notice_process_workflow" FORM_NUMBER_DAG_PARAM = "form_number" START_DATE_DAG_PARAM = "start_date" diff --git a/dags/selector_retransform_process_orchestrator.py b/dags/selector_retransform_process_orchestrator.py index 5ba321b53..e5ca395b4 100644 --- a/dags/selector_retransform_process_orchestrator.py +++ b/dags/selector_retransform_process_orchestrator.py @@ -13,7 +13,10 @@ DAG_NAME = "selector_re_transform_process_orchestrator" -RE_TRANSFORM_TARGET_NOTICE_STATES = [NoticeStatus.NORMALISED_METADATA, NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION] +RE_TRANSFORM_TARGET_NOTICE_STATES = [NoticeStatus.NORMALISED_METADATA, NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION, + NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION, NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION, + NoticeStatus.TRANSFORMED, NoticeStatus.DISTILLED + ] TRIGGER_NOTICE_PROCESS_WORKFLOW_TASK_ID = "trigger_notice_process_workflow" FORM_NUMBER_DAG_PARAM = "form_number" START_DATE_DAG_PARAM = "start_date" diff --git a/setup.py b/setup.py index 5f628a9c6..1961200dc 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ def open_local(paths, mode="r", encoding="utf8"): "metadata_generator = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_metadata_generator:main", "conceptual_mapping_differ = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_conceptual_mapping_differ:main", "rdf_differ = ted_sws.rdf_differ.entrypoints.cli.cmd_rdf_differ:main", - "mapping_suite_processor = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_processor:main", "yarrrml2rml_converter = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_yarrrml2rml_converter:main", "normalisation_resource_generator = ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources:main", diff --git a/ted_sws/__init__.py b/ted_sws/__init__.py index 8be595aab..97c128d1e 100644 --- a/ted_sws/__init__.py +++ b/ted_sws/__init__.py @@ -64,7 +64,7 @@ def AIRFLOW__CORE__EXECUTOR(self, config_value: str) -> str: def MONGO_DB_PORT(self, config_value: str) -> int: return int(config_value) - @env_property() + @env_property(default_value="aggregates_db") def MONGO_DB_AGGREGATES_DATABASE_NAME(self, config_value: str) -> str: return config_value @@ -115,7 +115,7 @@ def ELK_VERSION(self, config_value: str) -> int: class LoggingConfig: - @env_property() + @env_property(default_value="aggregates_db") def MONGO_DB_LOGS_DATABASE_NAME(self, config_value: str) -> str: return config_value @@ -151,7 +151,7 @@ class API: def ID_MANAGER_PROD_API_HOST(self, config_value: str) -> str: return config_value - @env_property(default_value="local_host") + @env_property(default_value="localhost") def ID_MANAGER_DEV_API_HOST(self, config_value: str) -> str: return config_value diff --git a/ted_sws/core/adapters/config_resolver.py b/ted_sws/core/adapters/config_resolver.py index f0ef9a864..e97810d8b 100644 --- a/ted_sws/core/adapters/config_resolver.py +++ b/ted_sws/core/adapters/config_resolver.py @@ -126,11 +126,11 @@ def env_property(config_resolver_class: Type[ConfigResolverABC] = EnvConfigResol """ def wrap(func): @property - def wrapped_f(self, *args, **kwargs): + def wrapped_function(self, *args, **kwargs): config_value = config_resolver_class().concrete_config_resolve(config_name=func.__name__, default_value=default_value) return func(self, config_value, *args, **kwargs) - return wrapped_f + return wrapped_function return wrap \ No newline at end of file diff --git a/ted_sws/core/model/notice.py b/ted_sws/core/model/notice.py index 49c755c48..ae8cb65d9 100644 --- a/ted_sws/core/model/notice.py +++ b/ted_sws/core/model/notice.py @@ -38,20 +38,28 @@ class NoticeStatus(IntEnum): """ RAW = 10 INDEXED = 15 + # STATES FOR RE-TRANSFORM ---BEGIN--- NORMALISED_METADATA = 20 INELIGIBLE_FOR_TRANSFORMATION = 23 # backlog status ELIGIBLE_FOR_TRANSFORMATION = 27 # forward status PREPROCESSED_FOR_TRANSFORMATION = 29 TRANSFORMED = 30 + # STATES FOR RE-VALIDATE---BEGIN--- DISTILLED = 35 + # STATES FOR RE-TRANSFORM ---END--- + # STATES FOR RE-VALIDATE---END--- + # STATES FOR RE-PACKAGE ---BEGIN--- VALIDATED = 40 INELIGIBLE_FOR_PACKAGING = 43 # backlog status ELIGIBLE_FOR_PACKAGING = 47 # forward status + # STATES FOR RE-PACKAGE ---END--- + # STATES FOR RE-PUBLISH ---BEGIN--- PACKAGED = 50 INELIGIBLE_FOR_PUBLISHING = 53 # backlog status ELIGIBLE_FOR_PUBLISHING = 57 # forward status + # STATES FOR RE-PUBLISH ---END--- PUBLISHED = 60 - PUBLICLY_UNAVAILABLE = 63 # to be investigated if more fine-grained checks can be adopted + PUBLICLY_UNAVAILABLE = 63 # to be investigated if more fine-grained checks can be adopted #TODO: Revalidate for public availability. PUBLICLY_AVAILABLE = 67 # forward status def __lt__(self, other): diff --git a/ted_sws/data_manager/adapters/mapping_suite_repository.py b/ted_sws/data_manager/adapters/mapping_suite_repository.py index 898251d82..8ad1b0887 100644 --- a/ted_sws/data_manager/adapters/mapping_suite_repository.py +++ b/ted_sws/data_manager/adapters/mapping_suite_repository.py @@ -31,16 +31,15 @@ class MappingSuiteRepositoryMongoDB(MappingSuiteRepositoryABC): """ _collection_name = "mapping_suite_collection" - _database_name = config.MONGO_DB_AGGREGATES_DATABASE_NAME or "aggregates_db" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name): + def __init__(self, mongodb_client: MongoClient, database_name: str = None): """ :param mongodb_client: :param database_name: """ mongodb_client = mongodb_client - self._database_name = database_name + self._database_name = database_name or config.MONGO_DB_AGGREGATES_DATABASE_NAME notice_db = mongodb_client[self._database_name] self.collection = notice_db[self._collection_name] diff --git a/ted_sws/data_manager/adapters/notice_repository.py b/ted_sws/data_manager/adapters/notice_repository.py index 89c76c858..81ffc96b8 100644 --- a/ted_sws/data_manager/adapters/notice_repository.py +++ b/ted_sws/data_manager/adapters/notice_repository.py @@ -21,6 +21,7 @@ NOTICE_TED_ID = "ted_id" NOTICE_STATUS = "status" NOTICE_CREATED_AT = "created_at" +NOTICE_ID = "notice_id" NOTICE_NORMALISED_METADATA = "normalised_metadata" NOTICE_PREPROCESSED_XML_MANIFESTATION = "preprocessed_xml_manifestation" NOTICE_DISTILLED_RDF_MANIFESTATION = "distilled_rdf_manifestation" @@ -28,6 +29,7 @@ NOTICE_METS_MANIFESTATION = "mets_manifestation" METADATA_PUBLICATION_DATE = "publication_date" METADATA_DOCUMENT_SENT_DATE = "document_sent_date" +FILE_STORAGE_COLLECTION_NAME = "fs.files" class NoticeRepositoryInFileSystem(NoticeRepositoryABC): @@ -122,18 +124,18 @@ class NoticeRepository(NoticeRepositoryABC): """ _collection_name = "notice_collection" - _database_name = config.MONGO_DB_AGGREGATES_DATABASE_NAME or "aggregates_db" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name): + def __init__(self, mongodb_client: MongoClient, database_name: str = None): + database_name = database_name if database_name else config.MONGO_DB_AGGREGATES_DATABASE_NAME self._database_name = database_name self.mongodb_client = mongodb_client notice_db = mongodb_client[self._database_name] self.file_storage = gridfs.GridFS(notice_db) self.collection = notice_db[self._collection_name] - self.collection.create_index([("created_at", ASCENDING)]) - self.collection.create_index([("status", ASCENDING)]) - self.file_storage_collection = notice_db["fs.files"] - self.file_storage_collection.create_index([("notice_id", ASCENDING)]) + self.collection.create_index([(NOTICE_CREATED_AT, ASCENDING)]) + self.collection.create_index([(NOTICE_STATUS, ASCENDING)]) + self.file_storage_collection = notice_db[FILE_STORAGE_COLLECTION_NAME] + self.file_storage_collection.create_index([(NOTICE_ID, ASCENDING)]) def get_file_content_from_grid_fs(self, file_id: str) -> str: """ @@ -169,7 +171,7 @@ def write_notice_fields_in_grid_fs(self, notice: Notice) -> Tuple[Notice, list, """ notice = copy.deepcopy(notice) linked_file_ids = [linked_file._id for linked_file in - self.file_storage.find({"notice_id": notice.ted_id})] + self.file_storage.find({NOTICE_ID: notice.ted_id})] new_linked_file_ids = [] diff --git a/ted_sws/data_manager/adapters/supra_notice_repository.py b/ted_sws/data_manager/adapters/supra_notice_repository.py index a458783ea..89f1f0d1a 100644 --- a/ted_sws/data_manager/adapters/supra_notice_repository.py +++ b/ted_sws/data_manager/adapters/supra_notice_repository.py @@ -16,10 +16,9 @@ class DailySupraNoticeRepository(DailySupraNoticeRepositoryABC): """ _collection_name = "daily_supra_notice_collection" - _database_name = config.MONGO_DB_AGGREGATES_DATABASE_NAME or "aggregates_db" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name): - self._database_name = database_name + def __init__(self, mongodb_client: MongoClient, database_name: str = None): + self._database_name = database_name or config.MONGO_DB_AGGREGATES_DATABASE_NAME self.mongodb_client = mongodb_client daily_supra_notice_db = mongodb_client[self._database_name] self.collection = daily_supra_notice_db[self._collection_name] diff --git a/ted_sws/event_manager/adapters/event_logging_repository.py b/ted_sws/event_manager/adapters/event_logging_repository.py index 4ca8a8b14..157b427c9 100644 --- a/ted_sws/event_manager/adapters/event_logging_repository.py +++ b/ted_sws/event_manager/adapters/event_logging_repository.py @@ -29,10 +29,9 @@ class EventLoggingRepository(EventLoggingRepositoryABC): """ This is the base/generic events' repository class. """ - _database_name = config.MONGO_DB_LOGS_DATABASE_NAME or "logs_db" _collection_name = "log_events" - def __init__(self, mongodb_client: MongoClient = None, database_name: str = _database_name, + def __init__(self, mongodb_client: MongoClient = None, database_name: str = None, collection_name: str = _collection_name): """ This is the constructor/initialization of base/generic event logging repository. @@ -41,7 +40,7 @@ def __init__(self, mongodb_client: MongoClient = None, database_name: str = _dat :param database_name: The database name :param collection_name: The collection name """ - self._database_name = database_name + self._database_name = database_name or config.MONGO_DB_LOGS_DATABASE_NAME self._collection_name = collection_name if mongodb_client is None: mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL) @@ -101,19 +100,14 @@ def add(self, event_message: EventMessage) -> str: result = self.collection.insert_one(record) return result.inserted_id - @classmethod - def get_default_database_name(cls): - return cls._database_name - class TechnicalEventRepository(EventLoggingRepository): """ This is the technical events' repository class. """ - _database_name = EventLoggingRepository._database_name _collection_name = "technical_events" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name, + def __init__(self, mongodb_client: MongoClient, database_name: str = None, collection_name: str = _collection_name): """ This is the constructor/initialization of technical event logging repository. @@ -129,10 +123,9 @@ class NoticeEventRepository(EventLoggingRepository): """ This is the notice events' repository class. """ - _database_name = EventLoggingRepository._database_name _collection_name = "notice_events" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name, + def __init__(self, mongodb_client: MongoClient, database_name: str = None, collection_name: str = _collection_name): """ This is the constructor/initialization of notice event logging repository. @@ -148,10 +141,9 @@ class MappingSuiteEventRepository(EventLoggingRepository): """ This is the mapping suite events' repository class. """ - _database_name = EventLoggingRepository._database_name _collection_name = "mapping_suite_events" - def __init__(self, mongodb_client: MongoClient, database_name: str = _database_name, + def __init__(self, mongodb_client: MongoClient, database_name: str = None, collection_name: str = _collection_name): """ This is the constructor/initialization of mapping suite event logging repository. diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ab293edec..da07cde96 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -4,6 +4,7 @@ import pymongo import pytest +from ted_sws import config from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC, SerializationFormat as RMLSerializationFormat from tests import TEST_DATA_PATH from tests.fakes.fake_rml_mapper import FakeRMLMapper @@ -43,4 +44,9 @@ def fake_mapping_suite_id() -> str: @pytest.fixture def invalid_mapping_suite_id() -> str: - return "test_invalid_package" \ No newline at end of file + return "test_invalid_package" + + +@pytest.fixture +def aggregates_database_name(): + return config.MONGO_DB_AGGREGATES_DATABASE_NAME diff --git a/tests/unit/data_manager/conftest.py b/tests/unit/data_manager/conftest.py index d8f59ab85..e7bb542d3 100644 --- a/tests/unit/data_manager/conftest.py +++ b/tests/unit/data_manager/conftest.py @@ -1,9 +1,5 @@ from datetime import date - -import mongomock -import pymongo import pytest - from ted_sws.core.model.supra_notice import DailySupraNotice from ted_sws.core.model.transform import MetadataConstraints, FileResource, TransformationRuleSet, SHACLTestSuite, \ SPARQLTestSuite, MappingSuite, TransformationTestData diff --git a/tests/unit/data_manager/test_mapping_suite_repository.py b/tests/unit/data_manager/test_mapping_suite_repository.py index 532e8fcdb..9010c703a 100644 --- a/tests/unit/data_manager/test_mapping_suite_repository.py +++ b/tests/unit/data_manager/test_mapping_suite_repository.py @@ -6,7 +6,7 @@ def test_mapping_suite_repository_mongodb(mongodb_client, fake_mapping_suite, - fake_mapping_suite_identifier_with_version): + fake_mapping_suite_identifier_with_version, aggregates_database_name): mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) mapping_suite_repository.add(mapping_suite=fake_mapping_suite) result_mapping_suite = mapping_suite_repository.get(reference=fake_mapping_suite_identifier_with_version) @@ -19,11 +19,12 @@ def test_mapping_suite_repository_mongodb(mongodb_client, fake_mapping_suite, assert result_mapping_suite.title == "updated_title" result_mapping_suites = list(mapping_suite_repository.list()) assert len(result_mapping_suites) == 1 - mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name) + mongodb_client.drop_database(aggregates_database_name) def test_mapping_suite_repository_mongodb_update_invalid_id(mongodb_client, fake_mapping_suite, - fake_mapping_suite_identifier_with_version): + fake_mapping_suite_identifier_with_version, + aggregates_database_name): mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) mapping_suite_repository.add(mapping_suite=fake_mapping_suite) result_mapping_suite = mapping_suite_repository.get(reference=fake_mapping_suite_identifier_with_version) @@ -34,7 +35,7 @@ def test_mapping_suite_repository_mongodb_update_invalid_id(mongodb_client, fake mapping_suite_repository.update(mapping_suite=result_mapping_suite) result_mapping_suite = mapping_suite_repository.get(reference=result_mapping_suite.identifier) assert result_mapping_suite is None - mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name) + mongodb_client.drop_database(aggregates_database_name) def test_mapping_suite_repository_in_file_system(file_system_repository_path, fake_mapping_suite): @@ -58,7 +59,8 @@ def test_mapping_suite_repository_in_file_system(file_system_repository_path, fa def test_inter_transactions_mapping_suite_repositories(mongodb_client, file_system_repository_path, fake_mapping_suite, - fake_mapping_suite_identifier_with_version): + fake_mapping_suite_identifier_with_version, + aggregates_database_name): mapping_suite_repository_mongodb = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) mapping_suite_repository_file_system = MappingSuiteRepositoryInFileSystem( repository_path=file_system_repository_path) @@ -69,4 +71,4 @@ def test_inter_transactions_mapping_suite_repositories(mongodb_client, file_syst result_mapping_suite = mapping_suite_repository_file_system.get(reference=fake_mapping_suite.identifier) assert DeepDiff(result_mapping_suite, fake_mapping_suite) == {} mapping_suite_repository_file_system.clear_repository() - mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name) + mongodb_client.drop_database(aggregates_database_name) diff --git a/tests/unit/data_manager/test_notice_repository.py b/tests/unit/data_manager/test_notice_repository.py index 53b4d3957..cccd7f286 100644 --- a/tests/unit/data_manager/test_notice_repository.py +++ b/tests/unit/data_manager/test_notice_repository.py @@ -9,8 +9,8 @@ NOTICE_TED_ID = "123456" -def test_notice_repository_create(mongodb_client): - mongodb_client.drop_database(NoticeRepository._database_name) +def test_notice_repository_create(mongodb_client, aggregates_database_name): + mongodb_client.drop_database(aggregates_database_name) notice_repository = NoticeRepository(mongodb_client=mongodb_client) notice = Notice(ted_id=NOTICE_TED_ID, original_metadata=TEDMetadata(**{"AA": ["Metadata"]}), xml_manifestation=XMLManifestation(object_data="HELLO")) @@ -29,11 +29,11 @@ def test_notice_repository_create(mongodb_client): assert result_notice assert result_notice.ted_id == NOTICE_TED_ID assert result_notice.original_metadata.AA == ["Updated metadata"] - mongodb_client.drop_database(NoticeRepository._database_name) + mongodb_client.drop_database(aggregates_database_name) -def test_notice_repository_get_notice_by_status(mongodb_client): - mongodb_client.drop_database(NoticeRepository._database_name) +def test_notice_repository_get_notice_by_status(mongodb_client, aggregates_database_name): + mongodb_client.drop_database(aggregates_database_name) notice_repository = NoticeRepository(mongodb_client=mongodb_client) notice = Notice(ted_id=NOTICE_TED_ID, original_metadata=TEDMetadata(**{"AA": ["Metadata"]}), xml_manifestation=XMLManifestation(object_data="HELLO")) @@ -43,9 +43,9 @@ def test_notice_repository_get_notice_by_status(mongodb_client): assert result_notice.status == NoticeStatus.RAW -def test_notice_repository_default_database_name(mongodb_client): +def test_notice_repository_default_database_name(mongodb_client, aggregates_database_name): notice_repository = NoticeRepository(mongodb_client=mongodb_client) - assert notice_repository._database_name == NoticeRepository._database_name + assert notice_repository._database_name == aggregates_database_name def test_notice_repository_create_notice_from_repository_result(mongodb_client): notice_repository = NoticeRepository(mongodb_client=mongodb_client) diff --git a/tests/unit/event_manager/adapters/event_logger/conftest.py b/tests/unit/event_manager/adapters/event_logger/conftest.py index 16a150bf1..7d53c1395 100644 --- a/tests/unit/event_manager/adapters/event_logger/conftest.py +++ b/tests/unit/event_manager/adapters/event_logger/conftest.py @@ -2,6 +2,7 @@ import pytest +from ted_sws import config from ted_sws.event_manager.adapters.event_handler import EventHandler, EventWriterToMongoDBHandler, \ EventWriterToFileHandler, EventWriterToConsoleHandler, EventWriterToNullHandler from ted_sws.event_manager.adapters.event_handler_config import ConsoleLoggerConfig @@ -120,7 +121,7 @@ def mongodb_handler(mongodb_client) -> EventWriterToMongoDBHandler: @pytest.fixture def logs_database_name() -> str: - return EventLoggingRepository.get_default_database_name() + return config.MONGO_DB_LOGS_DATABASE_NAME @pytest.fixture diff --git a/tests/unit/event_manager/adapters/event_logger/test_event_logging_repository.py b/tests/unit/event_manager/adapters/event_logger/test_event_logging_repository.py index f3157ac1c..9351e0806 100644 --- a/tests/unit/event_manager/adapters/event_logger/test_event_logging_repository.py +++ b/tests/unit/event_manager/adapters/event_logger/test_event_logging_repository.py @@ -5,8 +5,6 @@ def assert_event_repository(event_repository, event_message, logs_database_name): - assert EventLoggingRepository.get_default_database_name() - _id = event_repository.add(event_message) assert _id log: dict = event_repository.collection.find_one({"_id": ObjectId(_id)}) @@ -44,4 +42,3 @@ def test_mapping_suite_event_repository(logs_database_name, mapping_suite_event_ def test_event_logging_repository_wo_collection(mongodb_client, logs_database_name): with pytest.raises(ValueError): EventLoggingRepository(mongodb_client=mongodb_client, database_name=logs_database_name, collection_name='') - diff --git a/tests/unit/mapping_suite_processor/test_conceptual_mapping_processor.py b/tests/unit/mapping_suite_processor/test_conceptual_mapping_processor.py index 21590957f..b20e8c6ff 100644 --- a/tests/unit/mapping_suite_processor/test_conceptual_mapping_processor.py +++ b/tests/unit/mapping_suite_processor/test_conceptual_mapping_processor.py @@ -11,7 +11,7 @@ def test_mapping_suite_processor_upload_in_mongodb(file_system_repository_path, mongodb_client, - test_package_identifier_with_version): + test_package_identifier_with_version, aggregates_database_name): with temporary_copy(file_system_repository_path) as tmp_mapping_suite_package_path: mapping_suite_package_path = tmp_mapping_suite_package_path / "test_package" mapping_suite_processor_load_package_in_mongo_db(mapping_suite_package_path=mapping_suite_package_path, @@ -24,7 +24,7 @@ def test_mapping_suite_processor_upload_in_mongodb(file_system_repository_path, mapping_suite = mapping_suite_repository.get(reference=test_package_identifier_with_version) assert mapping_suite - mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name) + mongodb_client.drop_database(aggregates_database_name) def test_mapping_suite_processor_generate_metadata(file_system_repository_path): diff --git a/tests/unit/notice_transformer/test_notice_transformer.py b/tests/unit/notice_transformer/test_notice_transformer.py index f92a96fb7..425d9b6f1 100644 --- a/tests/unit/notice_transformer/test_notice_transformer.py +++ b/tests/unit/notice_transformer/test_notice_transformer.py @@ -32,7 +32,7 @@ def test_notice_transformer_by_id_function(fake_rml_mapper, mongodb_client, fake def test_notice_transformer_by_id_function_with_invalid_ids(fake_rml_mapper, mongodb_client, fake_mapping_suite, notice_2018, - notice_repository): + notice_repository, aggregates_database_name): notice_2018._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION notice_id = notice_2018.ted_id mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client) @@ -42,7 +42,7 @@ def test_notice_transformer_by_id_function_with_invalid_ids(fake_rml_mapper, mon result_notice = notice_repository.get(reference=notice_id) assert result_notice is None notice_repository.add(notice=notice_2018) - mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name) + mongodb_client.drop_database(aggregates_database_name) with pytest.raises(Exception): transform_notice_by_id(notice_id, fake_mapping_suite.identifier, notice_repository, mapping_suite_repository, fake_rml_mapper)