Extended tests and reference comparison

The changes include adding the following:
- Remote examples tests
- Pack tests
- Reference compare

Note: Currently this ./test/reference.md. The failed tests are yet to be
analysed.

.github/workflows/e2e_test.yaml

@@ -71,8 +71,13 @@ jobs:
 
       - name: Run Test
         shell: bash
+        continue-on-error: true
         run: |
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
+            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()
@@ -108,6 +113,7 @@ jobs:
 
     - name: Consolidate robot test results
       working-directory: artifacts
+      continue-on-error: true
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
@@ -117,7 +123,10 @@ jobs:
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md
+        python ./test/lib/execution_summary.py artifacts \
+          -r ./test/reference.md \
+          -o artifacts/collective_robot_results/output.xml \
+          -m summary_report.md
 
     - name: Print E2E Report
       if: always()

.github/workflows/nightly.yml

@@ -90,8 +90,8 @@ jobs:
         with:
           go-version-file: cpackget/go.mod
           cache-dependency-path: |
-            cpackget/go.mod
-            cpackget/go.sum
+            **/go.mod
+            **/go.sum
 
       - name: Build cpackget executable
         working-directory: cpackget
@@ -112,8 +112,8 @@ jobs:
         with:
           go-version-file: cbuild2cmake/go.mod
           cache-dependency-path: |
-            cbuild2cmake/go.mod
-            cbuild2cmake/go.sum
+            **/go.mod
+            **/go.sum
 
       - name: Build cbuild2cmake executable
         working-directory: cbuild2cmake
@@ -229,9 +229,14 @@ jobs:
 
       - name: Run Test
         shell: bash
+        continue-on-error: true
         run: |
           export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
+            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()
@@ -245,6 +250,7 @@ jobs:
     needs: [run-tests]
     runs-on: ubuntu-latest
     permissions: write-all
+
     steps:
     - uses: actions/checkout@v4
 
@@ -267,6 +273,7 @@ jobs:
 
     - name: Consolidate robot test results
       working-directory: artifacts
+      continue-on-error: true
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
@@ -276,7 +283,10 @@ jobs:
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md
+        python ./test/lib/execution_summary.py artifacts \
+          -r ./test/reference.md \
+          -o artifacts/collective_robot_results/output.xml \
+          -m summary_report.md
 
     - name: Print E2E Report
       if: always()

test/data/build-asm/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/build-c/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/build-cpp/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/include-define/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/language-scope/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/linker-pre-processing/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/pre-include/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/data/whitespace/solution.csolution.yml

@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

test/lib/elf_compare.py

@@ -2,11 +2,12 @@
 import subprocess
 from robot.api import logger
 from robot.utils import asserts
+from pathlib import Path
 
 def compare_elf_information(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir):
-    logger.debug('Input file: %s' % input_file)
-    logger.debug('Cbuildgen out dir: %s' % cbuildgen_out_dir)
-    logger.debug('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir)
+    logger.info('Input file: %s' % input_file)
+    logger.info('Cbuildgen out dir: %s' % cbuildgen_out_dir)
+    logger.info('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir)
     return CompareELF(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir).compare_elf_files()
 
 class Context:
@@ -34,7 +35,7 @@ class Utils:
     def run_command(exe_path, args):
         try:
             command = exe_path + ' ' + ' '.join(args)
-            logger.info(f"Running Command: {' '.join(command)}")
+            logger.info('Running Command: %s' % command)
             processOut = subprocess.run(command, check=True, shell=True, universal_newlines=True, capture_output=True, timeout=300)
             return True, processOut.stdout
         except subprocess.CalledProcessError as e:
@@ -66,6 +67,18 @@ def compare_elf_files(self):
             if path != '':
                 cbuildgen_elf_file = os.path.join(self.cbuildgen_out_dir, path)
                 cbuild2cmake_elf_file = os.path.join(self.cbuild2cmake_out_dir, path)
+                cbuildgenPath = Path(cbuildgen_elf_file)
+                cbuild2cmakePath = Path(cbuild2cmake_elf_file)
+                if cbuildgenPath.exists():
+                    logger.info('Path Exist: %s' % cbuildgenPath)
+                else:
+                    logger.info('Path doesnot Exist: %s' % cbuildgenPath)
+
+                if cbuild2cmakePath.exists():
+                    logger.info('Path Exist: %s' % cbuild2cmakePath)
+                else:
+                    logger.info('Path doesnot Exist: %s' % cbuild2cmakePath)
+
                 res1, stdout1 = self.get_elf_info(cbuildgen_elf_file)
                 res2, stdout2 = self.get_elf_info(cbuild2cmake_elf_file)
                 logger.info('Object Info %s' % stdout1)

test/lib/execution_summary.py

@@ -13,6 +13,7 @@
 from robot.api import ExecutionResult, ResultVisitor
 from robot.result.model import TestCase
 from robot.result.executionresult import Result 
+from reference_compare import *
 
 class ResultVisitorEx(ResultVisitor):
     def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str):
@@ -89,6 +90,7 @@ def end_result(self, result: Result):
             self.__write_test_section(f, self.failed_tests, "Failed Tests", "|Tag|Test|Message|:clock1030: Duration|Suite|\n")
             self.__write_test_section(f, self.skipped_tests, "Skipped Tests", "|Tag|Test|Suite|\n")
 
+
     def __write_test_section(self, file, test_dict, section_header, table_header):
         if len(test_dict) != 0:
             file.write(f"\n## {section_header}\n\n")
@@ -113,20 +115,21 @@ def __write_test_section(self, file, test_dict, section_header, table_header):
 def main():
     parser = argparse.ArgumentParser(description='Consolidate test summary report')
     parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files')
+    parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file')
     parser.add_argument('-o', '--output_file', type=str, nargs='?', default='output.xml', help='Path to output xml file')
     parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file')
     args = parser.parse_args()
 
-    test_env_files_path = args.test_env_files_path
-    output_file = args.output_file
-    markdown_file = args.markdown_file
+    # generate summary report
+    result = ExecutionResult(args.output_file)
+    result.visit(ResultVisitorEx(args.test_env_files_path, args.output_file, args.markdown_file))
 
-    result = ExecutionResult(output_file)
-    result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file))
+    # compare test execution summary and reference results 
+    return compare_test_results(args.markdown_file, args.reference_file)
 
 if __name__ == '__main__':
     try:
-        main()
+        sys.exit(main())
     except Exception as e:
         print(f'An error occurred: {e}', file=sys.stderr)
         sys.exit(1)

test/lib/reference_compare.py

@@ -0,0 +1,127 @@
+import argparse
+import sys
+import re
+
+class SummaryResult:
+    def __init__(self, passed, failed, skipped, total):
+        self.passed = passed
+        self.failed = failed
+        self.skipped = skipped
+        self.total = total
+
+    def __eq__(self, other):
+        if isinstance(other, SummaryResult):
+            return (self.passed == other.passed and
+                    self.failed == other.failed and
+                    self.skipped == other.skipped and
+                    self.total == other.total)
+        return False
+
+    def __str__(self):
+        return f'SummaryResult(passed={self.passed}, failed={self.failed}, skipped={self.skipped}, total={self.total})'
+
+class MarkdownReader:
+    def __init__(self, input_file_path: str):
+        self.input_file = input_file_path
+        self.content = ""
+
+    def read(self):
+        # Read the Markdown file
+        try:
+            with open(self.input_file, 'r') as file:
+                self.content = file.read()
+        except FileNotFoundError:
+            print(f"File not found: {self.input_file}", file=sys.stderr)
+            raise
+        except Exception as e:
+            print(f"Error reading file {self.input_file}: {e}", file=sys.stderr)
+            raise
+
+    def extract_summary(self):
+        self.read()
+        # Use regex to find the Summary section and extract the table values
+        summary_pattern = re.compile(
+            r"## Summary\n\n\|:white_check_mark: Passed\|:x: Failed\|:fast_forward: Skipped\|Total\|\n"
+            r"\|:----:\|:----:\|:-----:\|:---:\|\n\|(\d+)\|(\d+)\|(\d+)\|(\d+)\|"
+        )
+        match = summary_pattern.search(self.content)
+        if match:
+            return SummaryResult(*map(int, match.groups()))
+        else:
+            print(f"No summary found in file: {self.input_file}", file=sys.stderr)
+            return None
+
+    def extract_passed_tests(self):
+        self.read()
+
+        passed_tests_pattern = re.compile(
+            r"## Passed Tests\n\n\|Tag\|Test\|:clock1030: Duration\|Suite\|\n\|:---:\|:---:\|:---:\|:---:\|\n((\|.+\|\n)+)"
+        )
+
+        # Find the failed tests section
+        passed_tests_section = passed_tests_pattern.search(self.content)
+
+        # Extract the passed tests table if it exists
+        if not passed_tests_section:
+            return []
+
+        passed_tests_table = passed_tests_section.group(1)
+
+        # Regular expression to extract tag and test name from each row
+        # test_pattern = re.compile(r"\|(.+?)\|(.+?)\|.+?\|.+?\|.+?\|")
+        test_pattern = re.compile(r"(\|.+?\|.+?\|).+?\|.+?\|")
+
+        # Find all matches
+        passed_tests = test_pattern.findall(passed_tests_table)
+        return passed_tests
+
+def compare_test_results(markdown_file: str, reference_file: str):
+    md_file = MarkdownReader(markdown_file)
+    ref_file = MarkdownReader(reference_file)
+
+    md_summary = md_file.extract_summary()
+    ref_summary = ref_file.extract_summary()
+
+    if md_summary is None or ref_summary is None:
+        print("Comparison could not be performed due to missing summary data.", file=sys.stderr)
+        return False
+
+    error = 0
+    if md_summary != ref_summary:
+       print(f"error: Test results do not match the reference\n"
+          f"  Expected: Passed: {ref_summary.passed}, Failed: {ref_summary.failed}, Skipped: {ref_summary.skipped}, Total: {ref_summary.total}\n"
+          f"  Actual: Passed: {md_summary.passed}, Failed: {md_summary.failed}, Skipped: {md_summary.skipped}, Total: {md_summary.total}")
+       error = 1 # failure
+
+    md_passed_tests = md_file.extract_passed_tests()
+    ref_passed_tests = ref_file.extract_passed_tests()
+
+    for testInfo in md_passed_tests:
+        if testInfo in ref_passed_tests:
+            ref_passed_tests.remove(testInfo)
+
+    if len(ref_passed_tests) > 0:
+        print(f"error: Regression detected. The following tests were expected to execute and pass:")
+        for reg_test in ref_passed_tests:
+            print(f"  {reg_test}")
+        error = 1 # failure
+
+    return error
+
+def main():
+    parser = argparse.ArgumentParser(description='Compare test results in Markdown files.')
+    parser.add_argument('-r', '--reference_file', type=str, required=True, help='Path to reference file')
+    parser.add_argument('-m', '--markdown_file', type=str, default='summary_report.md', help='Path to consolidated summary markdown file')
+    args = parser.parse_args()
+
+    if compare_test_results(args.markdown_file, args.reference_file):
+        print("Regression found")
+    else:
+        print("No regression found")
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        print(f'An error occurred: {e}', file=sys.stderr)
+        sys.exit(1)

test/lib/utils.py

@@ -1,8 +1,8 @@
 import glob
 from pathlib import Path
-import shutil
 import subprocess
 import re
+# import os
 
 def glob_files_in_directory(directory:str, pattern: str, recursive: bool):
     yaml_files = glob.glob(directory + '/**/' + pattern, recursive=recursive)
@@ -13,7 +13,9 @@ def get_parent_directory_name(file_path:str):
     return parent_dir.name
 
 def get_parent_directory_path(file_path:str):
-    return Path(file_path).parent.absolute()
+    absPath = Path(file_path).parent.absolute()
+    path = str(absPath).replace('\\', '/')
+    return path
 
 def write_test_environment(test_env_file:str):
     toolList = ["cbuild", "cpackget", "csolution", "cbuild2cmake", "cbuildgen"]