MappingSuiteStructureValidator CLI-tool added

setup.py

@@ -87,6 +87,7 @@ def open_local(paths, mode="r", encoding="utf8"):
             "rml_modules_injector = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_rml_modules_injector:main",
             "yarrrml2rml_converter = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_yarrrml2rml_converter:main",
             "mapping_runner = ted_sws.notice_transformer.entrypoints.cli.cmd_mapping_runner:main",
+            "mapping_suite_validator = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_validator:main",
             "normalisation_resource_generator = ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources:main",
             "triple_store_loader = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_triple_store_loader:main",
             "bulk_packager = ted_sws.notice_packager.entrypoints.cli.cmd_bulk_packager:main",

ted_sws/mapping_suite_processor/adapters/mapping_suite_structure_checker.py

@@ -10,7 +10,7 @@
     MS_VALIDATE_FOLDER_NAME, MS_SPARQL_FOLDER_NAME, MS_SHACL_FOLDER_NAME, MS_OUTPUT_FOLDER_NAME, MS_TEST_SUITE_REPORT
 from ted_sws.event_manager.adapters.event_handler_config import ConsoleLoggerConfig
 from ted_sws.event_manager.adapters.event_logger import EventLogger
-from ted_sws.event_manager.model.event_message import EventMessage
+from ted_sws.event_manager.model.event_message import EventMessage, EventMessageLogSettings
 from ted_sws.event_manager.services.logger_from_context import get_env_logger
 from ted_sws.mapping_suite_processor.adapters.mapping_suite_hasher import MappingSuiteHasher
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import VERSION_FIELD, \
@@ -32,6 +32,7 @@ def __init__(self, mapping_suite_path: Union[pathlib.Path, str]):
         self.logger = get_env_logger(EventLogger(
             ConsoleLoggerConfig(name="MappingSuiteStructureValidator")
         ), is_cli=True)
+        self.log_settings = EventMessageLogSettings(**{"briefly": True})
 
     def assert_path(self, assertion_path_list: List[pathlib.Path]) -> bool:
         """
@@ -41,19 +42,22 @@ def assert_path(self, assertion_path_list: List[pathlib.Path]) -> bool:
         for path_item in assertion_path_list:
             message_path_not_found = f"Path not found: {path_item}"
             if not path_item.exists():
-                self.logger.error(event_message=EventMessage(message=message_path_not_found))
+                self.logger.error(event_message=EventMessage(message=message_path_not_found),
+                                  settings=self.log_settings)
                 success = False
                 continue
 
             if path_item.is_dir():
                 message_folder_empty = f"Folder is empty: {path_item}"
                 if not any(path_item.iterdir()):
-                    self.logger.error(event_message=EventMessage(message=message_folder_empty))
+                    self.logger.error(event_message=EventMessage(message=message_folder_empty),
+                                      settings=self.log_settings)
                     success = False
             else:
                 message_file_is_empty = f"File is empty: {path_item}"
-                if not path_item.stat().st_size > 0:
-                    self.logger.error(event_message=EventMessage(message=message_file_is_empty))
+                if path_item.stat().st_size <= 0:
+                    self.logger.error(event_message=EventMessage(message=message_file_is_empty),
+                                      settings=self.log_settings)
                     success = False
 
         return success
@@ -103,7 +107,8 @@ def _iter_dir(path):
                     report_count += 1
             if report_count < self.reports_min_count:
                 self.logger.error(
-                    event_message=EventMessage(message=f"{notice_path.stem} has missing validation reports."))
+                    event_message=EventMessage(message=f"{notice_path.stem} has missing validation reports."),
+                    settings=self.log_settings)
                 success = False
                 break
 
@@ -132,12 +137,12 @@ def check_metadata_consistency(self, package_metadata_path=None) -> bool:
         metadata_epo_version = [val for val in package_metadata.values()][4]
 
         if not (
-                conceptual_mappings_version > metadata_version
-                and conceptual_mappings_epo_version > metadata_epo_version
+                conceptual_mappings_version >= metadata_version
+                and conceptual_mappings_epo_version >= metadata_epo_version
         ):
             event_message = EventMessage(
                 message=f'Not the same value between metadata.json [version {metadata_version}, epo_version {metadata_epo_version}] and conceptual_mapping_file [version {conceptual_mappings_version}, epo_version {conceptual_mappings_epo_version}]')
-            self.logger.error(event_message=event_message)
+            self.logger.error(event_message=event_message, settings=self.log_settings)
             success = False
 
         return success
@@ -172,7 +177,8 @@ def check_for_changes_by_version(self) -> bool:
                         f'Conceptual Mappings ({version_in_cm}) '
                         f'does not correspond to the ones in the metadata.json file '
                         f'({metadata_json.get(MAPPING_SUITE_HASH)}, {metadata_json.get(VERSION_KEY)}). '
-                        f'Consider increasing the version and regenerating the metadata.json'))
+                        f'Consider increasing the version and regenerating the metadata.json'),
+                settings=self.log_settings)
             success = False
 
         return success

ted_sws/mapping_suite_processor/entrypoints/cli/cmd_mapping_suite_validator.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+from pathlib import Path
+
+import click
+
+from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
+from ted_sws.mapping_suite_processor.services.mapping_suite_validation_service import validate_mapping_suite
+
+CMD_NAME = "CMD_MAPPING_SUITE_VALIDATOR"
+
+"""
+USAGE:
+# mapping_suite_validator --help
+"""
+
+
+class CmdRunner(BaseCmdRunner):
+    """
+    Keeps the logic to be used by Mapping Suite Validator
+    """
+
+    def __init__(
+            self,
+            mapping_suite_id,
+            mappings_path
+    ):
+        super().__init__(name=CMD_NAME)
+        self.mapping_suite_id = mapping_suite_id
+        self.mappings_path = mappings_path
+
+    def run_cmd(self):
+        mapping_suite_path: Path = Path(self.mappings_path).resolve() / Path(self.mapping_suite_id)
+        is_valid: bool = validate_mapping_suite(mapping_suite_path)
+        self.run_cmd_result(Exception("Mapping Suite has an invalid structure") if not is_valid else None)
+
+
+def run(mapping_suite_id=None, opt_mappings_folder=DEFAULT_MAPPINGS_PATH):
+    cmd = CmdRunner(
+        mapping_suite_id=mapping_suite_id,
+        mappings_path=opt_mappings_folder
+    )
+    cmd.run()
+
+
+@click.command()
+@click.argument('mapping-suite-id', nargs=1, required=False)
+@click.option('-m', '--opt-mappings-folder', default=DEFAULT_MAPPINGS_PATH)
+def main(mapping_suite_id, opt_mappings_folder):
+    """
+    Validates a Mapping Suite (structure)
+    """
+    run(mapping_suite_id, opt_mappings_folder)
+
+
+if __name__ == '__main__':
+    main()

tests/unit/mapping_suite_processor/cli/test_cmd_mapping_suite_validator.py

@@ -0,0 +1,11 @@
+from ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_validator import main as cli_main
+from tests import TEST_DATA_PATH
+
+
+def test_mapping_suite_validator(cli_runner, mapping_suite_id):
+    response = cli_runner.invoke(cli_main, [mapping_suite_id, "--opt-mappings-folder",
+                                            TEST_DATA_PATH])
+    assert response.exit_code == 0
+    assert "SUCCESS" in response.output
+
+

tests/unit/mapping_suite_processor/conftest.py

@@ -39,18 +39,23 @@ def invalid_repository_path() -> str:
 
 
 @pytest.fixture
-def package_folder_path_for_validator():
-    return TEST_DATA_PATH / "package_F03_demo"
+def mapping_suite_id():
+    return "package_F03_demo"
 
 
 @pytest.fixture
-def conceptual_mappings_file_path():
-    return TEST_DATA_PATH / "package_F03_demo" / "transformation" / "conceptual_mappings.xlsx"
+def package_folder_path_for_validator(mapping_suite_id):
+    return TEST_DATA_PATH / mapping_suite_id
 
 
 @pytest.fixture
-def mapping_suite():
-    return TEST_DATA_PATH / "package_F03_demo"
+def conceptual_mappings_file_path(mapping_suite_id):
+    return TEST_DATA_PATH / mapping_suite_id / "transformation" / "conceptual_mappings.xlsx"
+
+
+@pytest.fixture
+def mapping_suite(mapping_suite_id):
+    return TEST_DATA_PATH / mapping_suite_id
 
 
 @pytest.fixture