From b92ea66521918d861d3708a9dead0023b6dabf3e Mon Sep 17 00:00:00 2001
From: Joey Vagedes <joey.vagedes@gmail.com>
Date: Wed, 1 May 2024 09:04:39 -0700
Subject: [PATCH] Host Based Unit Test updates

Updates the host based unit test runner to fail if a unit test
executable returns successfully, but has no test results, or if a test
suite generated from a unit test executable does not contain any tests.

The issues above indicate configuration errors in the unit test source
code itself and indicates to the developer that changes to the unit test
need to be made.

Updates the README.md file for the UnitTestFrameworkPkg to correct
inaccurate information regarding Code Coverage, and provide information
on how to consolidate and generate unit test html reports.
---
 .../HostBasedUnitTestRunner.py                | 11 +++++++-
 UnitTestFrameworkPkg/ReadMe.md                | 25 ++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py
index e7c37a1e792..439bbff3035 100644
--- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py
+++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py
@@ -117,10 +117,19 @@ def do_post_build(self, thebuilder):
                                  os.path.basename(test))
                     file_match_pattern = test + ".*." + arch + ".result.xml"
                     xml_results_list = glob.glob(file_match_pattern)
+                    if len(xml_results_list) == 0:
+                        logging.error(f'{os.path.basename(test)} did not generate any test suites.')
+                        logging.error('Review Code to ensure Test suites are created and tests are registered!')
+                        failure_count += 1
                     for xml_result_file in xml_results_list:
                         root = xml.etree.ElementTree.parse(
                             xml_result_file).getroot()
                         for suite in root:
+                            if len(suite) == 0:
+                                logging.error(f'TestSuite [{suite.attrib["name"]}] for test {test} did '
+                                              'not contain any test case.')
+                                logging.error('Review Code to ensure test cases are registered to the suite!')
+                                failure_count += 1
                             for case in suite:
                                 for result in case:
                                     if result.tag == 'failure':
@@ -130,7 +139,7 @@ def do_post_build(self, thebuilder):
                                             "  %s - %s" % (case.attrib['name'], result.text))
                                         failure_count += 1
 
-            if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE":
+            if thebuilder.env.GetValue("CODE_COVERAGE", "FALSE") == "TRUE":
                 if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5":
                     ret = self.gen_code_coverage_gcc(thebuilder)
                     if ret != 0:
diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md
index dd497617a93..4e05f442233 100644
--- a/UnitTestFrameworkPkg/ReadMe.md
+++ b/UnitTestFrameworkPkg/ReadMe.md
@@ -1491,6 +1491,18 @@ GTEST_OUTPUT=xml:<absolute or relative path to output file>
 
 This mode is used by the test running plugin to aggregate the results for CI test status reporting in the web view.
 
+### XML Reporting Test Consolidation
+
+There exists multiple tools for consolidating and generating consolidated test results from the
+test xml files that are generated. The arguably most convenient tool available is the
+`xunit-viewer` node package, installed via `npm install -g xunit-viewer`. This tool can
+consolidate all generated xml reports and create an html or cli summary of test results.
+
+The following command will generate a consolidated report at `<report_name>.html` and
+also print summary overview of the test results to the command line:
+
+`xunit-viewer --results Build --output <report_name>.html --console`
+
 ### Code Coverage
 
 Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report
@@ -1498,10 +1510,17 @@ per package tested, and combined cobertura report for all packages tested. The p
 present at `Build/<Pkg>/HostTest/<Target_Toolchain>/<Pkg>_coverage.xml`. The overall cobertura report will be present
 at `Build/coverage.xml`
 
-Code coverage generation has two config knobs:
+Code coverage generation has three config knobs. Each can be turned on/off by setting it to TRUE
+or FALSE e.g. `CC_REORGANIZE=TRUE`:
+
+1. `CC_REORGANIZE`: Controls if code coverage results are re-formatted into a "by-inf" folder
+   structure rather than the default "by-test" folder structure. Default: `TRUE`
+1. `CC_FULL`: Generates zero'd out coverage data for untested source files in the package. 
+   Default: `FALSE`
+2. `CC_FLATTEN`: Groups all source files together, rather than by INF. Default: `FALSE`
 
-1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package.
-2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF.
+** NOTE: `CC_FULL` and `CC_FLATTEN` values only matter if `CC_REORGANIZE=TRUE`, as they only
+effect how the coverage report is reorganized.
 
 **TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files
 that are consumed by multiple INFs.