Feature/ted 617

README.md

@@ -370,6 +370,30 @@ Options:
   --help                                      Show this message and exit.
 ```
 
+#### CMD: validation_summary_runner
+Generates Validation Summary for Notices.
+
+Use:
+```bash
+validation_summary_runner --help
+```
+to get the Usage Help:
+```bash
+Usage: validation_summary_runner [OPTIONS] MAPPING_SUITE_ID
+
+  Generates Validation Summary for Notices
+
+Options:
+  --notice-id TEXT
+  --notice-aggregate BOOLEAN
+  -m, --opt-mappings-folder TEXT
+  --help                                      Show this message and exit.
+```
+Use cases:
+1) If only MAPPING_SUITE_ID is provided: only mapping_suite validation_summary will be generated (if --notice-aggregate=1, validation_summary will also be generated for all notices)
+2) If --notice-id (LIST, multiple notice-ids can be provided) is also provided, validation_summary will be generated only for provided notices.
+
+
 #### CMD: rml_report_generator
 Generates RML modules report file for Mapping Suite.
 

dags/old_worker_single_notice_process_orchestrator.py

@@ -209,16 +209,17 @@ def _validate_transformed_rdf_manifestation(**context_args):
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
         mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+        validate_xpath_coverage_notice_by_id(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
+                                             mapping_suite_repository=mapping_suite_repository,
+                                             mongodb_client=mongodb_client)
         validate_notice_by_id_with_sparql_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
                                                 notice_repository=notice_repository,
                                                 mapping_suite_repository=mapping_suite_repository)
         validate_notice_by_id_with_shacl_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
                                                notice_repository=notice_repository,
                                                mapping_suite_repository=mapping_suite_repository
                                                )
-        validate_xpath_coverage_notice_by_id(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
-                                             mapping_suite_repository=mapping_suite_repository,
-                                             mongodb_client=mongodb_client)
+
         push_dag_downstream(NOTICE_ID, notice_id)
         push_dag_downstream(MAPPING_SUITE_ID, mapping_suite_id)
 

dags/worker_single_notice_process_orchestrator.py

@@ -196,9 +196,9 @@ def _validate_transformed_rdf_manifestation(**context_args):
         mapping_suite = pull_dag_upstream(MAPPING_SUITE_OBJECT)
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
 
+        validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, mongodb_client=mongodb_client)
         validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite)
         validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite)
-        validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, mongodb_client=mongodb_client)
         push_dag_downstream(NOTICE_OBJECT, notice)
         context = get_current_context()
         handle_event_message_metadata_dag_context(event_message, context)

requirements.txt

@@ -9,7 +9,6 @@ apache-airflow==2.2.5
 hvac==0.11.2
 SPARQLWrapper==1.8.5
 pandas==1.3.5
-pymessagebus~=1.2
 click~=8.1.0
 openpyxl==3.0.9
 colorama~=0.4

ted_sws/__init__.py

@@ -144,6 +144,11 @@ def ID_MANAGER_API_HOST(self) -> str:
         v: str = EnvConfigResolver().config_resolve()
         return v if v else "localhost"
 
+    @property
+    def ID_MANAGER_OLD_API_HOST(self) -> str:
+        v: str = EnvConfigResolver().config_resolve()
+        return v if v else "localhost"
+
     @property
     def ID_MANAGER_API_PORT(self) -> int:
         v: str = EnvConfigResolver().config_resolve()

ted_sws/core/adapters/cmd_runner.py

@@ -8,8 +8,7 @@
     EventWriterToMongoDBHandler
 from ted_sws.event_manager.adapters.event_handler_config import CLILoggerConfig
 from ted_sws.event_manager.adapters.event_logger import EventLogger
-from ted_sws.event_manager.adapters.log import SeverityLevelType
-from ted_sws.event_manager.adapters.logger import LOG_ERROR_TEXT, LOG_SUCCESS_TEXT
+from ted_sws.event_manager.adapters.log import SeverityLevelType, LOG_ERROR_TEXT, LOG_SUCCESS_TEXT
 from ted_sws.event_manager.model.event_message import EventMessage, EventMessageLogSettings
 from ted_sws.event_manager.services.logger_from_context import get_env_logger
 

ted_sws/core/model/manifestation.py

@@ -127,6 +127,7 @@ class SPARQLQuery(PropertyBaseModel):
     """
     title: Optional[str]
     description: Optional[str]
+    xpath: Optional[List[str]] = []
     query: str
 
 
@@ -136,6 +137,9 @@ class SPARQLQueryResult(PropertyBaseModel):
     """
     query: SPARQLQuery
     result: Optional[str]
+    query_result: Optional[str]
+    fields_covered: Optional[bool] = True
+    missing_fields: Optional[List[str]] = []
     error: Optional[str]
     identifier: Optional[str]
 

ted_sws/data_manager/entrypoints/cli/cmd_generate_mapping_resources.py

@@ -7,7 +7,7 @@
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
 from ted_sws.data_manager.adapters.mapping_suite_repository import TRANSFORM_PACKAGE_NAME, RESOURCES_PACKAGE_NAME
 from ted_sws.data_manager.adapters.sparql_endpoint import SPARQLTripleStoreEndpoint, TripleStoreEndpointABC
-from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
+from ted_sws.event_manager.adapters.log import LOG_INFO_TEXT
 from ted_sws.resources import QUERIES_PATH, MAPPING_FILES_PATH
 
 CELLAR_SPARQL_ENDPOINT_URL = "http://publications.europa.eu/webapi/rdf/sparql"

ted_sws/event_manager/adapters/log/__init__.py

@@ -4,6 +4,14 @@
 
 from ted_sws import RUN_ENV_NAME, RUN_TEST_ENV_VAL
 
+from colorama import Fore
+
+LOG_ERROR_TEXT = Fore.RED + "{}" + Fore.RESET
+LOG_SUCCESS_TEXT = Fore.GREEN + "{}" + Fore.RESET
+LOG_INFO_TEXT = Fore.CYAN + "{}" + Fore.RESET
+LOG_WARN_TEXT = Fore.YELLOW + "{}" + Fore.RESET
+
+
 EVENT_LOGGER_CONTEXT_KEY = '__event_logger__'
 
 
@@ -21,10 +29,5 @@ class ConfigHandlerType(Enum):
     MongoDBHandler = "MongoDBHandler"
 
 
-class LoggedBy(Enum):
-    DECORATOR = "decorator"
-    WRITER = "writer"
-
-
 def is_env_logging_enabled() -> bool:
     return os.environ[RUN_ENV_NAME] not in [RUN_TEST_ENV_VAL]