Feature/ted 661

.github/workflows/unit-tests-hermes.yml

@@ -16,19 +16,22 @@ jobs:
     runs-on: self-hosted
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
       - name: Make envfile
         run: make staging-dotenv-file
 
-      - name: Get RML mapper
-        run: make init-rml-mapper
-
       - name: Get Saxon
         run: make init-saxon
 
-      - name: Get Allure
-        run: make install-allure
+      - name: Run command
+        run: echo "Echo"
+
+      - name: Get RML mapper
+        run: make init-rml-mapper
+
+#      - name: Get Allure
+#        run: make install-allure
 
 #      - name: Start staging infra
 #        run: make start-project-services

Makefile

@@ -374,5 +374,5 @@ restore-mongodb:
 
 install-allure:
 	@ echo -e "Start install Allure commandline."
-	@ apt install npm
-	@ npm install -g allure-commandline
+	@ sudo apt -y install npm
+	@ sudo npm install -g allure-commandline

ted_sws/core/model/manifestation.py

@@ -250,6 +250,7 @@ class RDFManifestationValidationSummaryReport(PropertyBaseModel):
 
 
 class ValidationSummaryReport(ValidationManifestation):
+    notice_id: Optional[List[str]] = []
     xml_manifestation: Optional[XMLManifestationValidationSummaryReport] = XMLManifestationValidationSummaryReport()
     rdf_manifestation: Optional[RDFManifestationValidationSummaryReport] = RDFManifestationValidationSummaryReport()
     distilled_rdf_manifestation: Optional[

ted_sws/data_sampler/services/notice_xml_indexer.py

@@ -9,6 +9,8 @@
 from ted_sws.core.model.notice import Notice
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.resources import XSLT_FILES_PATH
+import xml.etree.ElementTree as XMLElementTree
+import re
 
 UNIQUE_XPATHS_XSLT_FILE_PATH = "get_unique_xpaths.xsl"
 XSLT_PREFIX_RESULT = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@@ -28,7 +30,7 @@ def index_notice_by_id(notice_id: str, mongodb_client: MongoClient):
     notice_repository.update(notice=notice)
 
 
-def index_notice(notice: Notice, xslt_transformer=None) -> Notice:
+def index_notice_xslt(notice: Notice, xslt_transformer=None) -> Notice:
     """
         This function selects unique XPath from XMlManifestation from a notice and indexes notices with these unique XPath.
     :param notice:
@@ -56,6 +58,51 @@ def index_notice(notice: Notice, xslt_transformer=None) -> Notice:
     return notice
 
 
+def index_notice(notice: Notice, base_xpath="") -> Notice:
+    def _notice_namespaces(xml_file) -> dict:
+        _namespaces = dict([node for _, node in XMLElementTree.iterparse(xml_file, events=['start-ns'])])
+        return {v: k for k, v in _namespaces.items()}
+
+    def _ns_tag(ns_tag):
+        tag = ns_tag[1]
+        ns = ns_tag[0]
+        if ns:
+            ns_alias = namespaces[ns]
+            if ns_alias:
+                return ns_alias + ":" + tag
+        return tag
+
+    def _xpath_generator(xml_file):
+        path = []
+        it = XMLElementTree.iterparse(xml_file, events=('start', 'end'))
+        for evt, el in it:
+            if evt == 'start':
+                ns_tag = re.split('[{}]', el.tag, 2)[1:]
+                path.append(_ns_tag(ns_tag) if len(ns_tag) > 1 else el.tag)
+
+                xpath = "/" + '/'.join(path)
+
+                if xpath.startswith(base_xpath + "/"):
+                    attributes = list(el.attrib.keys())
+                    if len(attributes) > 0:
+                        for attr in attributes:
+                            yield xpath + "/@" + attr
+                    yield xpath
+            else:
+                path.pop()
+
+    with tempfile.NamedTemporaryFile() as fp:
+        fp.write(notice.xml_manifestation.object_data.encode("utf-8"))
+
+        namespaces = _notice_namespaces(fp.name)
+        xpaths = list(set(_xpath_generator(fp.name)))
+        xml_metadata = XMLMetadata()
+        xml_metadata.unique_xpaths = xpaths
+        notice.set_xml_metadata(xml_metadata=xml_metadata)
+
+    return notice
+
+
 def get_unique_xpaths_from_notice_repository(mongodb_client: MongoClient) -> List[str]:
     """
         This function returns all unique XPaths in notice_repository.

ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py

@@ -5,6 +5,7 @@
 import pandas as pd
 
 from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_metadata
 
 VERSION_FIELD = 'Mapping Version'
 EPO_VERSION_FIELD = 'EPO version'
@@ -70,10 +71,7 @@ def mapping_suite_processor_generate_metadata(conceptual_mappings_file_path: pat
     :param output_metadata_file_path:
     :return:
     """
-    with open(conceptual_mappings_file_path, 'rb') as excel_file:
-        conceptual_mappings_metadata_df = pd.read_excel(excel_file, sheet_name=CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME)
-        raw_metadata = conceptual_mappings_metadata_df.set_index('Field').T.to_dict('list')
-        metadata = generate_metadata(raw_metadata=raw_metadata)
+    metadata = generate_metadata(raw_metadata=mapping_suite_read_metadata(conceptual_mappings_file_path))
 
     with open(output_metadata_file_path, 'w') as metadata_file:
         metadata_file.write(metadata)

ted_sws/notice_validator/adapters/validation_summary_runner.py

@@ -169,6 +169,7 @@ def validation_summary(cls, notices: List[Notice]) -> ValidationSummaryReport:
         report: ValidationSummaryReport = ValidationSummaryReport(
             object_data="ValidationSummaryReport"
         )
+        report.notice_id = list(map(lambda notice: notice.ted_id, notices))
 
         xml_manifestation_runner = XMLManifestationValidationSummaryRunner(notices)
         report.xml_manifestation = xml_manifestation_runner.validation_summary()

ted_sws/notice_validator/adapters/xpath_coverage_runner.py

@@ -31,11 +31,10 @@ class CoverageRunner:
     base_xpath: str
     mapping_suite_id: str
 
-    def __init__(self, mapping_suite_id: str, conceptual_mappings_file_path: PATH_TYPE = None, xslt_transformer=None,
+    def __init__(self, mapping_suite_id: str, conceptual_mappings_file_path: PATH_TYPE = None,
                  mongodb_client: MongoClient = None):
         self.mapping_suite_id = mapping_suite_id
         self.mongodb_client = mongodb_client
-        self.xslt_transformer = xslt_transformer
 
         conceptual_mapping: ConceptualMapping
         if self._db_readable():
@@ -124,7 +123,7 @@ def coverage_notice_xpath(self, notices: List[Notice], mapping_suite_id) -> XPAT
             if self._db_readable():
                 xpaths = get_unique_xpaths_covered_by_notices([notice.ted_id], self.mongodb_client)
             else:
-                notice = index_notice(notice, self.xslt_transformer)
+                notice = index_notice(notice)
 
                 if notice.xml_metadata and notice.xml_metadata.unique_xpaths:
                     xpaths = notice.xml_metadata.unique_xpaths

ted_sws/notice_validator/entrypoints/cli/cmd_xpath_coverage_runner.py

@@ -38,14 +38,12 @@ def __init__(
             self,
             mapping_suite_id,
             conceptual_mappings_file,
-            mappings_path,
-            xslt_transformer
+            mappings_path
     ):
         super().__init__(name=CMD_NAME)
         self.mapping_suite_id = mapping_suite_id
         self.mappings_path = mappings_path
         self.conceptual_mappings_file_path = Path(os.path.realpath(conceptual_mappings_file))
-        self.xslt_transformer = xslt_transformer
         self.output_folder = OUTPUT_FOLDER.format(mappings_path=self.mappings_path,
                                                   mapping_suite_id=self.mapping_suite_id)
 
@@ -79,8 +77,7 @@ def coverage_report(self, notices: List[Notice], output_path: Path, label: str):
         report = coverage_notice_xpath_report(notices,
                                               self.mapping_suite_id,
                                               self.conceptual_mappings_file_path,
-                                              self.coverage_runner,
-                                              self.xslt_transformer)
+                                              self.coverage_runner)
         self.save_json_report(Path(str(output_path) + ".json"), xpath_coverage_json_report(report))
         self.save_html_report(Path(str(output_path) + ".html"), xpath_coverage_html_report(report))
 
@@ -101,8 +98,7 @@ def run_cmd(self):
         return self.run_cmd_result()
 
 
-def run(mapping_suite_id=None, opt_conceptual_mappings_file=None, opt_mappings_folder=DEFAULT_MAPPINGS_PATH,
-        xslt_transformer=None):
+def run(mapping_suite_id=None, opt_conceptual_mappings_file=None, opt_mappings_folder=DEFAULT_MAPPINGS_PATH):
     if opt_conceptual_mappings_file:
         conceptual_mappings_file = opt_conceptual_mappings_file
     else:
@@ -114,8 +110,7 @@ def run(mapping_suite_id=None, opt_conceptual_mappings_file=None, opt_mappings_f
     cmd = CmdRunner(
         mapping_suite_id=mapping_suite_id,
         conceptual_mappings_file=conceptual_mappings_file,
-        mappings_path=opt_mappings_folder,
-        xslt_transformer=xslt_transformer
+        mappings_path=opt_mappings_folder
     )
     cmd.run()
 

ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2

@@ -10,7 +10,7 @@
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
     <title>HTML report for SPARQL Validation</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
-    <link rel="stylesheet" href="//code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
+    <link rel="stylesheet" href="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <style type="text/css">
         body {
             font-family: Arial, sans-serif;

ted_sws/notice_validator/resources/templates/validation_summary_report.jinja2

@@ -24,6 +24,38 @@
         table.display thead th {
             padding-left: 10px;
         }
+        [data-role=collapsible].collapsible-wrapper {
+            padding: 10px 0;
+        }
+        [data-role=collapsible] h4 {
+            margin: 0;
+        }
+        [data-role=collapsible] h4 a {
+            padding: 3px 10px;
+            background: #ccc;
+            width: 100%;
+            box-sizing: border-box;
+            display: block;
+            color: #111;
+            text-decoration: none;
+            margin: -3px 0;
+        }
+
+        [data-role=collapsible][data-state=collapsed] h4 a:before {
+            content: '+ expand';
+        }
+        [data-role=collapsible] h4 a:before {
+            content: '- collapse';
+        }
+        [data-role=collapsible][data-state=collapsed] .ui-collapsible-content {
+            visibility: hidden;
+            height: 0;
+            padding: 0 10px;;
+        }
+
+        [data-role=collapsible] .ui-collapsible-content {
+            padding: 10px;
+        }
     </style>
   </head>
   <body>
@@ -32,6 +64,19 @@
     <h2>Report details: </h2>
     <ul>
         <li>Date created: {{ created }}</li>
+        <li><div><hr></div>Notice identifier(s):
+            <div data-role="collapsible" data-state="{% if notice_id|length > 1 %}collapsed{% endif %}"
+                class="collapsible-wrapper">
+                <h4><a href="#"></a></h4>
+                <div class="ui-collapsible-content">
+                {% if notice_id|length == 1 %}
+                    {{ notice_id[0] }}
+                {% else %}
+                    {{ notice_id | tojson(indent=2) | replace('\n', '<br>') }}
+                {% endif %}
+                </div>
+            </div>
+        </li>
     </ul>
     <hr>
     <h2>XML Manifestation</h2>
@@ -191,7 +236,7 @@
     </table>
     {% endfor %}
     <p>-------</p>
-    <h3>SPARQL Summary</h3>
+    <h3>SHACL Summary</h3>
     <h4>Result severity: </h4>
     <ul>
         <li>Info: {{ manifestation.shacl_summary.result_severity.aggregate.info }}</li>
@@ -246,6 +291,13 @@
 <script type="text/javascript" src="https://cdn.datatables.net/buttons/1.6.4/js/buttons.print.min.js"></script>
 <script type="text/javascript">
     $(document).ready(function () {
+        $("[data-role=collapsible] h4 a").on("click", function(e) {
+            $p = $(this).closest("[data-role=collapsible]");
+            $p.attr("data-state", $p.attr("data-state") == "collapsed" ? "expanded" : "collapsed");
+            $c = $p.find('.ui-collapsible-content');
+            $c.attr("data-state", $c.attr("data-state") == "collapsed" ? "expanded" : "collapsed");
+            return false;
+        });
         $("table.display").DataTable({
             dom: 'B<"clear">t',
             buttons: [],

ted_sws/notice_validator/services/xpath_coverage_runner.py

@@ -25,11 +25,10 @@ def generate_report(self) -> XPATHCoverageValidationReport:
 
 def coverage_notice_xpath_report(notices: List[Notice], mapping_suite_id,
                                  conceptual_mappings_file_path: PATH_TYPE = None,
-                                 coverage_runner: CoverageRunner = None, xslt_transformer=None,
+                                 coverage_runner: CoverageRunner = None,
                                  mongodb_client: MongoClient = None) -> XPATHCoverageValidationReport:
     if not coverage_runner:
-        coverage_runner = CoverageRunner(mapping_suite_id, conceptual_mappings_file_path, xslt_transformer,
-                                         mongodb_client)
+        coverage_runner = CoverageRunner(mapping_suite_id, conceptual_mappings_file_path, mongodb_client)
     report: XPATHCoverageValidationReport = coverage_runner.coverage_notice_xpath(notices, mapping_suite_id)
     return report
 

ted_sws/supra_notice_manager/services/supra_notice_validator.py

@@ -42,7 +42,7 @@ def validate_and_update_daily_supra_notice(notice_publication_day: day_type, mon
     repo.update(daily_supra_notice=supra_notice)
 
 
-def validate_and_update_daily_supra_notice_manifestation(notice_publication_day: day_type, mongodb_client: MongoClient):
+def summary_validation_for_daily_supra_notice(notice_publication_day: day_type, mongodb_client: MongoClient):
     if isinstance(notice_publication_day, date):
         notice_publication_day = datetime.combine(notice_publication_day, time())
 
@@ -61,4 +61,5 @@ def validate_and_update_daily_supra_notice_manifestation(notice_publication_day:
             notices.append(notice)
 
     supra_notice.validation_summary = generate_validation_summary_report_notices(notices)
+    supra_notice.validation_summary.notice_id = []  # no notice_ids needed to be stored for supra_notice
     repo.update(daily_supra_notice=supra_notice)

tests/e2e/data_sampler/test_unique_xpaths_from_xml.py

@@ -1,7 +1,7 @@
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.data_sampler.services.notice_selectors import get_notice_ids_by_form_number, \
     get_notice_ids_by_eforms_subtype
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, index_notice_by_id, \
+from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, index_notice_xslt, index_notice_by_id, \
     get_unique_xpaths_from_notice_repository, get_unique_notice_id_from_notice_repository, \
     get_minimal_set_of_notices_for_coverage_xpaths, get_minimal_set_of_xpaths_for_coverage_notices, \
     get_unique_notices_id_covered_by_xpaths, get_unique_xpaths_covered_by_notices, get_most_representative_notices
@@ -12,6 +12,11 @@ def test_index_notice(notice_2016):
     assert len(result_notice.xml_metadata.unique_xpaths) == 112
 
 
+def test_index_notice_xslt(notice_2016):
+    result_notice = index_notice_xslt(notice=notice_2016)
+    assert len(result_notice.xml_metadata.unique_xpaths) == 112
+
+
 def test_index_notice_by_id(notice_2016, mongodb_client):
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     notice_repository.add(notice=notice_2016)

tests/unit/notice_validator/cli/test_cmd_xpath_coverage_runner.py

@@ -30,26 +30,22 @@ def post_process(fake_repository_path, fake_mapping_suite_id):
     assert len(report_files) == 2
 
 
-def test_cmd_xpath_coverage_runner(caplog, fake_mapping_suite_F03_id, fake_repository_path,
-                                   fake_xslt_transformer):
+def test_cmd_xpath_coverage_runner(caplog, fake_mapping_suite_F03_id, fake_repository_path):
     cli_run(
         mapping_suite_id=fake_mapping_suite_F03_id,
-        opt_mappings_folder=fake_repository_path,
-        xslt_transformer=fake_xslt_transformer
+        opt_mappings_folder=fake_repository_path
     )
 
     assert "SUCCESS" in caplog.text
 
     post_process(fake_repository_path, fake_mapping_suite_F03_id)
 
 
-def test_cmd_xpath_coverage_runner_with_invalid_input(caplog, fake_repository_path, invalid_mapping_suite_id,
-                                                      fake_xslt_transformer):
+def test_cmd_xpath_coverage_runner_with_invalid_input(caplog, fake_repository_path, invalid_mapping_suite_id):
     with pytest.raises(FileNotFoundError):
         cli_run(
             mapping_suite_id=invalid_mapping_suite_id,
             opt_mappings_folder=fake_repository_path,
-            opt_conceptual_mappings_file="invalid",
-            xslt_transformer=fake_xslt_transformer
+            opt_conceptual_mappings_file="invalid"
         )
         assert "FAILED" in caplog.text