Fix!/fix certain naming issues, better patterns for multitests with parts

doc/newsfragments/2626_changed.pattern_with_parts.rst

@@ -0,0 +1,4 @@
+Filtering Patterns now are easier to use with MultiTest with parts.
+
+    * Support filtering MultiTest with both part-specified patterns and part-unspecified patterns.
+    * Applying filtering patterns will not change testcase shuffling - the same testcase will always end up in the same part of a MultiTest.

examples/Multitest/Tagging and Filtering/Composite Filters/test_plan_command_line.py

@@ -4,10 +4,9 @@
 """
 import sys
 
-from testplan.testing.multitest import MultiTest, testsuite, testcase
-
 from testplan import test_plan
 from testplan.report.testing.styles import Style
+from testplan.testing.multitest import MultiTest, testcase, testsuite
 
 
 @testsuite
@@ -96,8 +95,8 @@ def test_2(self, env, result):
 
 # Run all tests: tagged with `server`
 # AND (belong to `Gamma` multitest OR has the name `test_3`)
-# command line: `--tags server --pattern Gamma *:*:test_3`
-# command line (alt.): `--tags server --pattern Gamma --pattern *:*:test_3`
+# command line: `--tags server --patterns Gamma *:*:test_3`
+# command line (alt.): `--tags server --patterns Gamma --patterns *:*:test_3`
 
 
 @test_plan(

testplan/common/report/base.py

@@ -7,16 +7,16 @@
 Later on these reports would be merged together to
 build the final report as the testplan result.
 """
-import copy
-import time
-import uuid
 import collections
+import copy
 import dataclasses
 import itertools
-
+import time
+import uuid
 from typing import Dict, List, Optional, Union
 
 from testplan.common.utils import strings
+
 from .log import create_logging_adapter
 
 
@@ -57,11 +57,17 @@ class Report:
     exception_logger = ExceptionLogger
 
     def __init__(
-        self, name, description=None, uid=None, entries=None, parent_uids=None
+        self,
+        name: str,
+        description: Optional[str] = None,
+        definition_name: Optional[str] = None,
+        uid: Optional[str] = None,
+        entries: Optional[list] = None,
+        parent_uids: Optional[List[str]] = None,
     ):
         self.name = name
         self.description = description
-
+        self.definition_name = definition_name or name
         self.uid = uid or name
         self.entries = entries or []
 
@@ -77,12 +83,12 @@ def __init__(
         self.parent_uids = parent_uids or []
 
     def __str__(self):
-        return '{kls}(name="{name}", id="{uid}")'.format(
+        return '{kls}(name="{name}", uid="{uid}")'.format(
             kls=self.__class__.__name__, name=self.name, uid=self.uid
         )
 
     def __repr__(self):
-        return '{kls}(name="{name}", id="{uid}", entries={entries})'.format(
+        return '{kls}(name="{name}", uid="{uid}", entries={entries})'.format(
             kls=self.__class__.__name__,
             name=self.name,
             uid=self.uid,
@@ -131,15 +137,15 @@ def logged_exceptions(self, *exception_classes, **kwargs):
 
     def _check_report(self, report):
         """
-        Utility method for checking `report` `type` and `uid`.
+        Utility method for checking `report` `type` and `definition_name`.
         """
         msg = "Report check failed for `{}` and `{}`. ".format(self, report)
 
-        if report.uid != self.uid:
+        if report.definition_name != self.definition_name:
             raise AttributeError(
                 msg
-                + "`uid` attributes (`{}`, `{}`) do not match.".format(
-                    self.uid, report.uid
+                + "`definition_name` attributes (`{}`, `{}`) do not match.".format(
+                    self.definition_name, report.definition_name
                 )
             )
 

testplan/common/report/schemas.py

@@ -4,10 +4,10 @@
 from marshmallow import Schema, fields, post_load
 from marshmallow.utils import EXCLUDE
 
-from testplan.common.serialization import schemas, fields as custom_fields
-
-from .base import Report, ReportGroup, EventRecorder
+from testplan.common.serialization import fields as custom_fields
+from testplan.common.serialization import schemas
 
+from .base import EventRecorder, Report, ReportGroup
 
 __all__ = ["ReportLogSchema", "ReportSchema", "ReportGroupSchema"]
 
@@ -36,6 +36,9 @@ class Meta:
 
     name = fields.String()
     description = fields.String(allow_none=True)
+    definition_name = fields.String(
+        allow_none=True
+    )  # otherwise new tpr cannot process old report
     entries = fields.List(custom_fields.NativeOrPretty())
     parent_uids = fields.List(fields.String())
 

testplan/common/utils/interface.py

@@ -1,7 +1,7 @@
 """Validates methods signature."""
 
 from inspect import signature
-from typing import Union, Optional
+from typing import Union
 
 
 class NoSuchMethodInClass(Exception):

testplan/common/utils/testing.py

@@ -1,22 +1,22 @@
 """This module contains utilites for testing Testplan itself."""
 
-import sys
+import collections
 import functools
+import io
 import logging
-import pprint
 import os
-import io
+import pprint
+import sys
 import warnings
+from contextlib import contextmanager
+from typing import Collection
 
 from lxml import objectify
 
-from contextlib import contextmanager
-from typing import Collection
 from testplan.runners.pools.tasks.base import Task
 
 from ..report.base import Report, ReportGroup
 from ..utils.comparison import is_regex
-import collections
 
 null_handler = logging.NullHandler()
 
@@ -287,6 +287,7 @@ def check_report_context(report, ctx):
     interested in report contents, just the existence of reports
     with matching names, with the correct order.
     """
+    assert len(report) == len(ctx)
     for mt_report, (multitest_name, suite_ctx) in zip(report, ctx):
         assert mt_report.name == multitest_name
         assert len(mt_report) == len(suite_ctx)

testplan/exporters/testing/coverage/__init__.py

@@ -7,19 +7,20 @@
 import sys
 from collections import OrderedDict
 from contextlib import contextmanager
-from typing import Dict, Generator, Mapping, TextIO, Tuple, Optional
+from typing import Dict, Generator, Mapping, Optional, TextIO, Tuple
 
 from testplan.common.exporters import (
-    ExporterConfig,
     ExportContext,
+    ExporterConfig,
     verify_export_context,
 )
 from testplan.exporters.testing.base import Exporter
 from testplan.report.testing.base import (
-    TestCaseReport,
+    ReportCategories,
     TestGroupReport,
     TestReport,
 )
+from testplan.testing.common import TEST_PART_PATTERN_FORMAT_STRING
 
 
 class CoveredTestsExporter(Exporter):
@@ -52,8 +53,11 @@ def export(
             # here we use an OrderedDict as an ordered set
             results = OrderedDict()
             for entry in source.entries:
-                if isinstance(entry, TestGroupReport):
-                    self._append_covered_group_n_case(entry, [], results)
+                if (
+                    isinstance(entry, TestGroupReport)
+                    and entry.category == ReportCategories.MULTITEST
+                ):
+                    self._append_mt_coverage(entry, results)
             if results:
                 with _custom_open(self.cfg.tracing_tests_output) as (
                     f,
@@ -68,27 +72,44 @@ def export(
             return None
         return None
 
-    def _append_covered_group_n_case(
+    def _append_mt_coverage(
         self,
         report: TestGroupReport,
-        path: Tuple[str, ...],
         result: Mapping[Tuple[str, ...], None],
     ):
         """
-        Recursively add test group or test case with covered_lines set to
-        the result ordered set.
+        Add test entity with covered_lines set to the result ordered set.
 
         Here we use an OrderedDict as an ordered set.
         """
-        curr_path = (*path, report.name)
+
+        if report.part is not None:
+            mt_pat = TEST_PART_PATTERN_FORMAT_STRING.format(
+                report.definition_name, report.part[0], report.part[1]
+            )
+        else:
+            mt_pat = report.definition_name
+
         if report.covered_lines:
-            result[curr_path] = None
-        for entry in report.entries:
-            if isinstance(entry, TestGroupReport):
-                self._append_covered_group_n_case(entry, curr_path, result)
-            elif isinstance(entry, TestCaseReport):
-                if entry.covered_lines:
-                    result[(*curr_path, entry.name)] = None
+            result[(mt_pat,)] = None
+        for st in report.entries:
+            if st.covered_lines:
+                result[(mt_pat, st.definition_name)] = None
+            for tc in st.entries:
+                if tc.category == ReportCategories.PARAMETRIZATION:
+                    for sub_tc in tc.entries:
+                        if sub_tc.covered_lines:
+                            result[
+                                (
+                                    mt_pat,
+                                    st.definition_name,
+                                    sub_tc.definition_name,
+                                )
+                            ] = None
+                elif tc.covered_lines:
+                    result[
+                        (mt_pat, st.definition_name, tc.definition_name)
+                    ] = None
 
 
 @contextmanager

testplan/exporters/testing/pdf/renderers/reports.py

@@ -3,25 +3,26 @@
 """
 import logging
 from collections import OrderedDict
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 
 from reportlab.lib import colors, styles
 from reportlab.platypus import Paragraph
 
 from testplan.common.exporters.pdf import RowStyle
 from testplan.common.utils.registry import Registry
-from testplan.common.utils.strings import format_description, wrap, split_text
+from testplan.common.utils.strings import format_description, split_text, wrap
 from testplan.report import (
+    ReportCategories,
     Status,
-    TestReport,
-    TestGroupReport,
     TestCaseReport,
-    ReportCategories,
+    TestGroupReport,
+    TestReport,
 )
 from testplan.report.testing.styles import StyleFlag
 from testplan.testing import tagging
+
 from . import constants as const
-from .base import format_duration, RowData, BaseRowRenderer, MetadataMixin
+from .base import BaseRowRenderer, MetadataMixin, RowData, format_duration
 
 
 class ReportRendererRegistry(Registry):
@@ -260,6 +261,9 @@ def get_header(
 
         [<TEST_NAME> - <NATIVE TAGS>][][][<TEST_STATUS>]
 
+        This method is also used by its subclass, where source will be of type
+        ``TestCaseReport``.
+
         :param source: Source object for the renderer.
         :param depth: Depth of the source object on report tree. Used for indentation.
         :param row_idx: Index of the current table row to be rendered.
@@ -417,7 +421,7 @@ def get_header_linestyle(self) -> Tuple[int, colors.HexColor]:
 
     def get_header(
         self,
-        source: TestCaseReport,
+        source: TestGroupReport,
         depth: int,
         row_idx: int,
     ) -> RowData:

testplan/parser.py

@@ -6,9 +6,11 @@
 import copy
 import json
 import sys
+import warnings
 from typing import Dict, List
 
 import schema
+
 from testplan import defaults
 from testplan.common.utils import logger
 from testplan.report.testing import (
@@ -219,7 +221,7 @@ def generate_parser(self) -> HelpParser:
 
 --patterns <Multitest Name>
 --patterns <Multitest Name 1> <Multitest Name 2>
---patterns <Multitest Name 1> --pattern <Multitest Name 2>
+--patterns <Multitest Name 1> --patterns <Multitest Name 2>
 --patterns <Multitest Name>:<Suite Name>
 --patterns <Multitest Name>:<Suite Name>:<Testcase name>
 --patterns <Multitest Name>:*:<Testcase name>
@@ -511,6 +513,15 @@ def process_args(self, namespace: argparse.Namespace) -> Dict:
         if args["list"] and not args["test_lister"]:
             args["test_lister"] = listing.NameLister()
 
+        if (
+            args["interactive_port"] is not None
+            and args["tracing_tests"] is not None
+        ):
+            warnings.warn(
+                "Tracing tests feature not available in interactive mode."
+            )
+            args["tracing_tests"] = None
+
         return args