diff --git a/setup.py b/setup.py index 028c8029e..36abf0447 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ def open_local(paths, mode="r", encoding="utf8"): "rml_modules_injector = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_rml_modules_injector:main", "yarrrml2rml_converter = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_yarrrml2rml_converter:main", "mapping_runner = ted_sws.notice_transformer.entrypoints.cli.cmd_mapping_runner:main", + "mapping_suite_validator = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_validator:main", "normalisation_resource_generator = ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources:main", "triple_store_loader = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_triple_store_loader:main", "bulk_packager = ted_sws.notice_packager.entrypoints.cli.cmd_bulk_packager:main", diff --git a/ted_sws/mapping_suite_processor/adapters/mapping_suite_structure_checker.py b/ted_sws/mapping_suite_processor/adapters/mapping_suite_structure_checker.py index 64fcd891b..343a69239 100644 --- a/ted_sws/mapping_suite_processor/adapters/mapping_suite_structure_checker.py +++ b/ted_sws/mapping_suite_processor/adapters/mapping_suite_structure_checker.py @@ -10,7 +10,7 @@ MS_VALIDATE_FOLDER_NAME, MS_SPARQL_FOLDER_NAME, MS_SHACL_FOLDER_NAME, MS_OUTPUT_FOLDER_NAME, MS_TEST_SUITE_REPORT from ted_sws.event_manager.adapters.event_handler_config import ConsoleLoggerConfig from ted_sws.event_manager.adapters.event_logger import EventLogger -from ted_sws.event_manager.model.event_message import EventMessage +from ted_sws.event_manager.model.event_message import EventMessage, EventMessageLogSettings from ted_sws.event_manager.services.logger_from_context import get_env_logger from ted_sws.mapping_suite_processor.adapters.mapping_suite_hasher import MappingSuiteHasher from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import VERSION_FIELD, \ @@ -32,6 +32,7 @@ def __init__(self, mapping_suite_path: Union[pathlib.Path, str]): self.logger = get_env_logger(EventLogger( ConsoleLoggerConfig(name="MappingSuiteStructureValidator") ), is_cli=True) + self.log_settings = EventMessageLogSettings(**{"briefly": True}) def assert_path(self, assertion_path_list: List[pathlib.Path]) -> bool: """ @@ -41,19 +42,22 @@ def assert_path(self, assertion_path_list: List[pathlib.Path]) -> bool: for path_item in assertion_path_list: message_path_not_found = f"Path not found: {path_item}" if not path_item.exists(): - self.logger.error(event_message=EventMessage(message=message_path_not_found)) + self.logger.error(event_message=EventMessage(message=message_path_not_found), + settings=self.log_settings) success = False continue if path_item.is_dir(): message_folder_empty = f"Folder is empty: {path_item}" if not any(path_item.iterdir()): - self.logger.error(event_message=EventMessage(message=message_folder_empty)) + self.logger.error(event_message=EventMessage(message=message_folder_empty), + settings=self.log_settings) success = False else: message_file_is_empty = f"File is empty: {path_item}" - if not path_item.stat().st_size > 0: - self.logger.error(event_message=EventMessage(message=message_file_is_empty)) + if path_item.stat().st_size <= 0: + self.logger.error(event_message=EventMessage(message=message_file_is_empty), + settings=self.log_settings) success = False return success @@ -103,7 +107,8 @@ def _iter_dir(path): report_count += 1 if report_count < self.reports_min_count: self.logger.error( - event_message=EventMessage(message=f"{notice_path.stem} has missing validation reports.")) + event_message=EventMessage(message=f"{notice_path.stem} has missing validation reports."), + settings=self.log_settings) success = False break @@ -132,12 +137,12 @@ def check_metadata_consistency(self, package_metadata_path=None) -> bool: metadata_epo_version = [val for val in package_metadata.values()][4] if not ( - conceptual_mappings_version > metadata_version - and conceptual_mappings_epo_version > metadata_epo_version + conceptual_mappings_version >= metadata_version + and conceptual_mappings_epo_version >= metadata_epo_version ): event_message = EventMessage( message=f'Not the same value between metadata.json [version {metadata_version}, epo_version {metadata_epo_version}] and conceptual_mapping_file [version {conceptual_mappings_version}, epo_version {conceptual_mappings_epo_version}]') - self.logger.error(event_message=event_message) + self.logger.error(event_message=event_message, settings=self.log_settings) success = False return success @@ -172,7 +177,8 @@ def check_for_changes_by_version(self) -> bool: f'Conceptual Mappings ({version_in_cm}) ' f'does not correspond to the ones in the metadata.json file ' f'({metadata_json.get(MAPPING_SUITE_HASH)}, {metadata_json.get(VERSION_KEY)}). ' - f'Consider increasing the version and regenerating the metadata.json')) + f'Consider increasing the version and regenerating the metadata.json'), + settings=self.log_settings) success = False return success diff --git a/ted_sws/mapping_suite_processor/entrypoints/cli/cmd_mapping_suite_validator.py b/ted_sws/mapping_suite_processor/entrypoints/cli/cmd_mapping_suite_validator.py new file mode 100755 index 000000000..36fc494fc --- /dev/null +++ b/ted_sws/mapping_suite_processor/entrypoints/cli/cmd_mapping_suite_validator.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 + +from pathlib import Path + +import click + +from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH +from ted_sws.mapping_suite_processor.services.mapping_suite_validation_service import validate_mapping_suite + +CMD_NAME = "CMD_MAPPING_SUITE_VALIDATOR" + +""" +USAGE: +# mapping_suite_validator --help +""" + + +class CmdRunner(BaseCmdRunner): + """ + Keeps the logic to be used by Mapping Suite Validator + """ + + def __init__( + self, + mapping_suite_id, + mappings_path + ): + super().__init__(name=CMD_NAME) + self.mapping_suite_id = mapping_suite_id + self.mappings_path = mappings_path + + def run_cmd(self): + mapping_suite_path: Path = Path(self.mappings_path).resolve() / Path(self.mapping_suite_id) + is_valid: bool = validate_mapping_suite(mapping_suite_path) + self.run_cmd_result(Exception("Mapping Suite has an invalid structure") if not is_valid else None) + + +def run(mapping_suite_id=None, opt_mappings_folder=DEFAULT_MAPPINGS_PATH): + cmd = CmdRunner( + mapping_suite_id=mapping_suite_id, + mappings_path=opt_mappings_folder + ) + cmd.run() + + +@click.command() +@click.argument('mapping-suite-id', nargs=1, required=False) +@click.option('-m', '--opt-mappings-folder', default=DEFAULT_MAPPINGS_PATH) +def main(mapping_suite_id, opt_mappings_folder): + """ + Validates a Mapping Suite (structure) + """ + run(mapping_suite_id, opt_mappings_folder) + + +if __name__ == '__main__': + main() diff --git a/tests/unit/mapping_suite_processor/cli/test_cmd_mapping_suite_validator.py b/tests/unit/mapping_suite_processor/cli/test_cmd_mapping_suite_validator.py new file mode 100644 index 000000000..26145be45 --- /dev/null +++ b/tests/unit/mapping_suite_processor/cli/test_cmd_mapping_suite_validator.py @@ -0,0 +1,11 @@ +from ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_validator import main as cli_main +from tests import TEST_DATA_PATH + + +def test_mapping_suite_validator(cli_runner, mapping_suite_id): + response = cli_runner.invoke(cli_main, [mapping_suite_id, "--opt-mappings-folder", + TEST_DATA_PATH]) + assert response.exit_code == 0 + assert "SUCCESS" in response.output + + diff --git a/tests/unit/mapping_suite_processor/conftest.py b/tests/unit/mapping_suite_processor/conftest.py index 4cd956d52..94f33097d 100644 --- a/tests/unit/mapping_suite_processor/conftest.py +++ b/tests/unit/mapping_suite_processor/conftest.py @@ -39,18 +39,23 @@ def invalid_repository_path() -> str: @pytest.fixture -def package_folder_path_for_validator(): - return TEST_DATA_PATH / "package_F03_demo" +def mapping_suite_id(): + return "package_F03_demo" @pytest.fixture -def conceptual_mappings_file_path(): - return TEST_DATA_PATH / "package_F03_demo" / "transformation" / "conceptual_mappings.xlsx" +def package_folder_path_for_validator(mapping_suite_id): + return TEST_DATA_PATH / mapping_suite_id @pytest.fixture -def mapping_suite(): - return TEST_DATA_PATH / "package_F03_demo" +def conceptual_mappings_file_path(mapping_suite_id): + return TEST_DATA_PATH / mapping_suite_id / "transformation" / "conceptual_mappings.xlsx" + + +@pytest.fixture +def mapping_suite(mapping_suite_id): + return TEST_DATA_PATH / mapping_suite_id @pytest.fixture