Skip to content

Commit

Permalink
Conceptual Mappings Differ
Browse files Browse the repository at this point in the history
  • Loading branch information
Kolea Plesco committed Oct 6, 2022
1 parent fe8b7d6 commit ae357b1
Show file tree
Hide file tree
Showing 9 changed files with 112 additions and 60 deletions.
21 changes: 12 additions & 9 deletions docs/antora/modules/ROOT/pages/mapping_suite_cli_toolchain.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -494,13 +494,14 @@ Usage: conceptual_mapping_differ [OPTIONS]
Generate reports (JSON, HTML) with differences between 2 Conceptual Mappings
Options:
--mapping-suite-id TEXT Mapping Suite IDs
--file TEXT Conceptual Mappings files
--branch TEXT GIT branches or tags
-ms-id, --mapping-suite-id TEXT Mapping Suite IDs
-f, --file TEXT Conceptual Mappings files
-b, --branch TEXT GIT branches or tags
-m, --opt-mappings-folder TEXT
-o, --opt-output-folder TEXT
--help Show this message and exit.
----
Use for:

Expand All @@ -515,12 +516,14 @@ Use for:
* --mapping-suite-id vs --mapping-suite-id
# conceptual_mapping_differ --mapping-suite-id=<MAPPING_SUITE_ID1> --mapping-suite-id=<MAPPING_SUITE_ID2>
* --branch vs --mapping-suite-id
# conceptual_mapping_differ --branch=<BRANCH1> --mapping-suite-id=<MAPPING_SUITE_ID2>
* --branch + --mapping-suite-id vs --branch + --mapping-suite-id
# conceptual_mapping_differ --branch=<BRANCH1> --mapping-suite-id=<MAPPING_SUITE_ID1> --branch=<BRANCH2> --mapping-suite-id=<MAPPING_SUITE_ID2>
# conceptual_mapping_differ -b <BRANCH1> -ms-id <MAPPING_SUITE_ID1> -b <BRANCH2> -ms-id <MAPPING_SUITE_ID2>
* --branch + --mapping-suite-id vs --file
# conceptual_mapping_differ --branch=<BRANCH1> --mapping-suite-id=<MAPPING_SUITE_ID1> --file=<FILE2>
* --branch vs --file
# conceptual_mapping_differ --branch=<BRANCH1> --file=<FILE2>
* --branch + --mapping-suite-id (remote) vs --mapping-suite-id (local)
# conceptual_mapping_differ --branch=<BRANCH> --mapping-suite-id=<MAPPING_SUITE_ID>
* --branch vs --branch
# conceptual_mapping_differ --branch=<BRANCH1> --branch=<BRANCH2>
----
4 changes: 2 additions & 2 deletions ted_sws/core/adapters/cmd_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,15 @@ def __init__(self, name=__name__, logger: EventLogger = None):
self.logger = logger

@classmethod
def _init_list_input_opts_split(cls, input_val) -> List:
def _init_list_input_opts_split(cls, input_val: List[str]) -> List:
input_list = []
if input_val and len(input_val) > 0:
for item in input_val:
input_list += map(lambda x: x.strip(), item.split(","))
return input_list

@classmethod
def _init_list_input_opts(cls, input_val) -> List:
def _init_list_input_opts(cls, input_val: List[str]) -> List:
"""
This method takes command line arguments (with multiple values), each element of which can have
comma separated values and generate a list from all the values, also removing duplicates.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from pathlib import Path

import click
from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT

from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
from ted_sws.mapping_suite_processor.entrypoints.cli import CONCEPTUAL_MAPPINGS_FILE_TEMPLATE
Expand Down Expand Up @@ -49,7 +50,7 @@ def __init__(
self.mappings_path = mappings_path
self.output_folder = output_folder

def _report(self, data):
def _report(self, data, files: list):
report_file_file_name_json = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".json")
with open(report_file_file_name_json, 'w+') as report_file:
report_file.write(json.dumps(data, indent=2))
Expand All @@ -59,64 +60,84 @@ def _report(self, data):
generate_conceptual_mappings_diff_html_report(
ConceptualMappingDiff(
metadata={
"files": self.file,
"branches": self.branch,
"mapping_suite_ids": self.mapping_suite_id,
"branches": self.branch
"files": files
},
data=data
))
)

@classmethod
def _conceptual_mappings_file_path(cls, mappings_path, mapping_suite_id):
return CONCEPTUAL_MAPPINGS_FILE_TEMPLATE.format(
mappings_path=mappings_path,
mapping_suite_id=mapping_suite_id
)

def _mappings_path(self) -> Path:
mappings_path = Path(self.mappings_path).resolve()
assert Path(mappings_path).is_dir()
return mappings_path

def _display_input(self):
if self.branch:
self.log(LOG_WARN_TEXT.format("GIT Branches: ") + str(self.branch))
if self.mapping_suite_id:
self.log(LOG_WARN_TEXT.format("MappingSuites: ") + str(self.mapping_suite_id))
if self.file:
self.log(LOG_WARN_TEXT.format("Files: ") + str(self.file))

def run_cmd(self):
self._display_input()

diff = {}
filepath1 = None
filepath2 = None

file_len = len(self.file)
if self.file and file_len == 2:
filepath1 = self.file[0]
assert Path(filepath1).is_file()
filepath2 = self.file[1]
assert Path(filepath2).is_file()
elif self.mapping_suite_id:
mappings_path = Path(self.mappings_path).resolve()
assert Path(mappings_path).is_dir()

mapping_suite_id_len = len(self.mapping_suite_id)

if mapping_suite_id_len == 2:
filepath1 = CONCEPTUAL_MAPPINGS_FILE_TEMPLATE.format(
mappings_path=mappings_path,
mapping_suite_id=self.mapping_suite_id[0]
)
assert Path(filepath1).is_file()
filepath2 = CONCEPTUAL_MAPPINGS_FILE_TEMPLATE.format(
mappings_path=mappings_path,
mapping_suite_id=self.mapping_suite_id[1]
)
assert Path(filepath2).is_file()
elif mapping_suite_id_len == 1 and file_len == 1:
filepath1 = CONCEPTUAL_MAPPINGS_FILE_TEMPLATE.format(
mappings_path=mappings_path,
mapping_suite_id=self.mapping_suite_id[0]
)
mapping_suite_id_len = len(self.mapping_suite_id)
branch_len = len(self.branch)

if not self.branch:
if self.file and file_len == 2:
filepath1 = self.file[0]
assert Path(filepath1).is_file()
filepath2 = self.file[0]
filepath2 = self.file[1]
assert Path(filepath2).is_file()
elif self.mapping_suite_id:
mappings_path = self._mappings_path()
if mapping_suite_id_len == 2:
filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
assert Path(filepath1).is_file()
filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[1])
assert Path(filepath2).is_file()
elif mapping_suite_id_len == 1 and file_len == 1:
filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
assert Path(filepath1).is_file()
filepath2 = self.file[0]
assert Path(filepath2).is_file()

error = None
if filepath1 and filepath2:
diff = mapping_suite_diff_files_conceptual_mappings([Path(filepath1), Path(filepath2)])
elif self.branch:
mapping_suite_diff_repo_conceptual_mappings(
assert mapping_suite_id_len > 0
if branch_len == 1 and mapping_suite_id_len == 1 and not self.file:
mappings_path = self._mappings_path()
filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
else:
filepath2 = (self.file[0] if file_len == 1 else None)

diff = mapping_suite_diff_repo_conceptual_mappings(
branch_or_tag_name=self.branch,
mapping_suite_id=self.mapping_suite_id,
filepath=(Path(self.file[0]) if file_len == 1 else None)
filepath=Path(filepath2) if filepath2 else None
)
else:
error = Exception("Cannot do a diff with provided input!")

self._report(data=diff)
self._report(data=diff, files=[filepath1, filepath2])
self.run_cmd_result(error)


Expand All @@ -133,14 +154,38 @@ def run(mapping_suite_id=None, file=None, branch=None, opt_mappings_folder=DEFAU


@click.command()
@click.option('--mapping-suite-id', multiple=True, required=False, help="Mapping Suite IDs")
@click.option('--file', multiple=True, required=False, help="Conceptual Mappings files")
@click.option('--branch', multiple=True, required=False, help="GIT branches or tags")
@click.option('-ms-id', '--mapping-suite-id', multiple=True, required=False, help="Mapping Suite IDs")
@click.option('-f', '--file', multiple=True, required=False, help="Conceptual Mappings files")
@click.option('-b', '--branch', multiple=True, required=False, help="GIT branches or tags")
@click.option('-m', '--opt-mappings-folder', default=DEFAULT_MAPPINGS_PATH)
@click.option('-o', '--opt-output-folder', default=DEFAULT_REPORT_OUTPUT_FOLDER)
def main(mapping_suite_id, file, branch, opt_mappings_folder, opt_output_folder):
"""
Generate reports (JSON, HTML) with differences between 2 Conceptual Mappings
---
* --file vs --file:
# conceptual_mapping_differ --file=<CONCEPTUAL_MAPPINGS_FILE1> --file=<CONCEPTUAL_MAPPINGS_FILE2>
* --mapping-suite-id vs --file:
# conceptual_mapping_differ --mapping-suite-id=<MAPPING_SUITE_ID1> --file=<CONCEPTUAL_MAPPINGS_FILE2>
* --mapping-suite-id vs --mapping-suite-id:
# conceptual_mapping_differ --mapping-suite-id=<MAPPING_SUITE_ID1> --mapping-suite-id=<MAPPING_SUITE_ID2>
* --branch + --mapping-suite-id vs --branch + --mapping-suite-id:
# conceptual_mapping_differ --branch=<BRANCH1> --mapping-suite-id=<MAPPING_SUITE_ID1> --branch=<BRANCH2> --mapping-suite-id=<MAPPING_SUITE_ID2>
# conceptual_mapping_differ -b <BRANCH1> -ms-id <MAPPING_SUITE_ID1> -b <BRANCH2> -ms-id <MAPPING_SUITE_ID2>
* --branch + --mapping-suite-id vs --file:
# conceptual_mapping_differ --branch=<BRANCH1> --mapping-suite-id=<MAPPING_SUITE_ID1> --file=<FILE2>
* --branch + --mapping-suite-id (remote) vs --mapping-suite-id (local):
# conceptual_mapping_differ --branch=<BRANCH> --mapping-suite-id=<MAPPING_SUITE_ID>
---
"""
return run(mapping_suite_id, file, branch, opt_mappings_folder, opt_output_folder)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ def mapping_suite_diff_files_conceptual_mappings(filepaths: List[Path]) -> dict:
:return:
"""
assert filepaths and len(filepaths) == 2
assert filepaths[0].is_file()
assert filepaths[1].is_file()
return mapping_suite_diff_conceptual_mappings([
mapping_suite_read_conceptual_mapping(filepaths[0]),
mapping_suite_read_conceptual_mapping(filepaths[1])
Expand All @@ -54,7 +56,6 @@ def mapping_suite_diff_repo_conceptual_mappings(branch_or_tag_name: List[str], m
1) repo vs file
2) repo vs repo
:param github_repository_url:
:param mapping_suite_id:
:param branch_or_tag_name:
:param filepath:
Expand All @@ -79,6 +80,7 @@ def mapping_suite_diff_repo_conceptual_mappings(branch_or_tag_name: List[str], m
filepath1 = Path(temp_file1.name)

if filepath:
assert filepath.is_file()
filepath2 = filepath
else:
if len(branch_or_tag_name) < 2:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ def sparql_validation_generator(data: pd.DataFrame, base_xpath: str, controlled_

def _process_concept_mapping_sheet(sheet: pd.DataFrame) -> pd.DataFrame:
sheet.columns = sheet.iloc[0]
sheet = sheet[1:]
return sheet
return sheet[1:].copy()


def mapping_suite_processor_generate_sparql_queries(conceptual_mappings_file_path: pathlib.Path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@ def _read_list_from_pd_multiline_value(value: str) -> list:


def _df_to_dict(df: pd.DataFrame, key: str) -> dict:
return df.set_index(key).T.to_dict('list')
return df.copy().set_index(key).T.to_dict('list')


def _df_to_list(df: pd.DataFrame) -> list:
return df.tolist()
return df.copy().tolist()


def mapping_suite_read_metadata(conceptual_mappings_file_path: Path) -> Dict:
Expand Down Expand Up @@ -104,8 +104,7 @@ def _read_conceptual_mapping_rules(df: pd.DataFrame) -> List[ConceptualMappingRu
"""

df.columns = df.iloc[0]
rules_df = df[1:]

rules_df = df[1:].copy()
rules_df[RULES_SF_FIELD_ID].ffill(axis="index", inplace=True)
rules_df[RULES_SF_FIELD_NAME].ffill(axis="index", inplace=True)

Expand Down
2 changes: 0 additions & 2 deletions ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ def __init__(
repository_path = Path(self.mappings_path)
mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=repository_path)
self.mapping_suite = mapping_suite_repository.get(reference=self.mapping_suite_id)
self.log("K :: " + str(repository_path))


@classmethod
def save_report(cls, report_path, report_name, report_id, content):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@ def test_cmd_conceptual_mapping_differ(caplog, cli_runner, fake_test_mapping_sui

response = cli_runner.invoke(cli_main,
["--mapping-suite-id", fake_test_mapping_suite_id, "--mapping-suite-id",
fake_test_mapping_suite_id,
"--opt-mappings-folder", temp_mapping_suite_path, "--opt-output-folder",
temp_folder])
fake_test_mapping_suite_id, "--opt-mappings-folder", temp_mapping_suite_path,
"--opt-output-folder", temp_folder])

assert response.exit_code == 0
assert "SUCCESS" in response.output
Expand All @@ -52,3 +51,10 @@ def test_cmd_conceptual_mapping_differ(caplog, cli_runner, fake_test_mapping_sui
assert "FAILED" in response.output
assert "Cannot do a diff" in response.output

response = cli_runner.invoke(cli_main,
["--branch", "main", "--mapping-suite-id", "package_F03_test",
"--opt-mappings-folder", temp_mapping_suite_path, "--opt-output-folder",
temp_folder])

assert response.exit_code == 0
assert "SUCCESS" in response.output
Binary file not shown.

0 comments on commit ae357b1

Please sign in to comment.