Assign a data path and serial number to each test in discover

To support both multihost scenarios and/or running a test multiple
times, tmt needs to distinguish test data directories of different test
instances. To allow this, each test instance is given a "serial number"
which we can use in a data directory name.

Note: there can be more elaborate ways for picking the serial numbers,
the patch starts with something really trivial yet functioning.

Another note: the patch does not isolate tests on a phase level, a
single test, executed on different guests, may still land in the same
data directory. Different patch will solve this issue.

tests/discover/main.fmf

@@ -56,3 +56,7 @@
 /force:
     summary: Force run of discover finds tests
     test: ./force.sh
+
+/serial-number:
+    summary: Test test serial number assignment
+    test: ./serial-number.sh

tests/discover/serial-number-root-test/.fmf/version

@@ -0,0 +1 @@
+1

tests/discover/serial-number-root-test/main.fmf

@@ -0,0 +1 @@
+test: /bin/true

tests/discover/serial-number.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "workdir=\$(mktemp -d)" 0 "Create tmp directory"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Single discover phase"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/single-discover'"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/single-discover/discover/tests.yaml"
+        rlAssertGrep "/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/tests/foo 2" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Multiple discover phases"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/multiple-discover'"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/multiple-discover/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+        rlAssertGrep "/default-2/tests/bar 5" $rlRun_LOG
+        rlAssertGrep "/default-2/tests/foo 6" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Multiple plans"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/multiple-plans'"
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/multiple-plans/plan1/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/multiple-plans/plan2/discover/tests.yaml"
+        rlAssertGrep "/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/tests/foo 2" $rlRun_LOG
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/multiple-plans/plan3/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Single '/' test"
+        rlRun 'pushd serial-number-root-test'
+        rlRun "tmt -vv run --scratch --id $workdir discover -h fmf"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serial_number)\"' $workdir/plans/default/discover/tests.yaml"
+        rlAssertGrep "/ 1" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        # rlRun "rm -rf $workdir" 0 'Remove tmp directory'
+    rlPhaseEnd
+rlJournalEnd

tests/discover/serial-number/.fmf/version

@@ -0,0 +1 @@
+1

tests/discover/serial-number/plans.fmf

@@ -0,0 +1,29 @@
+execute:
+    how: tmt
+provision:
+    how: local
+
+/single-discover:
+    discover:
+        how: fmf
+
+/multiple-discover:
+    discover:
+        - how: fmf
+        - how: fmf
+        - how: fmf
+
+/multiple-plans:
+    /plan1:
+        discover:
+          - how: fmf
+          - how: fmf
+
+    /plan2:
+        discover:
+          how: fmf
+
+    /plan3:
+        discover:
+          - how: fmf
+          - how: fmf

tests/discover/serial-number/tests.fmf

@@ -0,0 +1,5 @@
+/foo:
+  test: /bin/true
+
+/bar:
+  test: /bin/true

tests/unit/test_report_junit.py

@@ -128,7 +128,7 @@ def assert_xml(actual_filepath, expected):
 class TestStateMapping:
     def test_pass(self, report_fix):
         report, results, out_file_path = report_fix
-        results.append(Result(result=ResultOutcome.PASS, name="/pass"))
+        results.append(Result(result=ResultOutcome.PASS, name="/pass", serial_number=1))
 
         report.go()
 
@@ -142,7 +142,7 @@ def test_pass(self, report_fix):
 
     def test_info(self, report_fix):
         report, results, out_file_path = report_fix
-        results.append(Result(result=ResultOutcome.INFO, name="/info"))
+        results.append(Result(result=ResultOutcome.INFO, name="/info", serial_number=1))
         report.go()
 
         assert_xml(out_file_path, """<?xml version="1.0" ?>
@@ -157,7 +157,7 @@ def test_info(self, report_fix):
 
     def test_warn(self, report_fix):
         report, results, out_file_path = report_fix
-        results.append(Result(result=ResultOutcome.WARN, name="/warn"))
+        results.append(Result(result=ResultOutcome.WARN, name="/warn", serial_number=1))
         report.go()
 
         assert_xml(out_file_path, """<?xml version="1.0" ?>
@@ -172,7 +172,7 @@ def test_warn(self, report_fix):
 
     def test_error(self, report_fix):
         report, results, out_file_path = report_fix
-        results.append(Result(result=ResultOutcome.ERROR, name="/error"))
+        results.append(Result(result=ResultOutcome.ERROR, name="/error", serial_number=1))
         report.go()
 
         assert_xml(out_file_path, """<?xml version="1.0" ?>
@@ -187,7 +187,7 @@ def test_error(self, report_fix):
 
     def test_fail(self, report_fix):
         report, results, out_file_path = report_fix
-        results.append(Result(result=ResultOutcome.FAIL, name="/fail"))
+        results.append(Result(result=ResultOutcome.FAIL, name="/fail", serial_number=1))
         report.go()
 
         assert_xml(out_file_path, """<?xml version="1.0" ?>

tmt/base.py

@@ -3,14 +3,15 @@
 import copy
 import dataclasses
 import enum
+import itertools
 import os
 import re
 import shutil
 import sys
 import time
 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Generator,
-                    Iterable, List, Optional, Sequence, Tuple, TypeVar, Union,
-                    cast)
+                    Iterable, Iterator, List, Optional, Sequence, Tuple,
+                    TypeVar, Union, cast)
 
 import fmf
 import fmf.base
@@ -702,6 +703,8 @@ class Test(Core, tmt.export.Exportable['Test']):
     duration: str = DEFAULT_TEST_DURATION_L1
     result: str = 'respect'
 
+    serial_number: int = 0
+
     returncode: Optional[int] = None
     real_duration: Optional[str] = None
     _reboot_count: int = 0
@@ -1177,6 +1180,22 @@ def _expand_node_data(self, data: T, fmf_context: Dict[str, str]) -> T:
                 data[i] = self._expand_node_data(item, fmf_context)
         return data
 
+    # TODO: better, more elaborete ways of assigning serial numbers to tests
+    # can be devised - starting with a really trivial one: each test gets
+    # one, starting with `1`.
+    #
+    # For now, the test itself is not important, and it's part of the method
+    # signature to leave the door open for more sophisticated methods that
+    # might depend on the actual test properties. Our simple "increment by 1"
+    # method does not need it.
+    _test_serial_number_generator: Optional[Iterator[int]] = None
+
+    def draw_test_serial_number(self, test: Test) -> int:
+        if self._test_serial_number_generator is None:
+            self._test_serial_number_generator = itertools.count(start=1, step=1)
+
+        return next(self._test_serial_number_generator)
+
     @property
     def environment(self) -> EnvironmentType:
         """ Return combined environment from plan data and command line """

tmt/result.py

@@ -76,6 +76,7 @@ class Result(tmt.utils.SerializableContainer):
     """ Describes what tmt knows about a single test result """
 
     name: str
+    serial_number: int
     result: ResultOutcome = field(
         default=ResultOutcome.PASS,
         serialize=lambda result: result.value,
@@ -141,6 +142,7 @@ def from_test(
 
         _result = Result(
             name=test.name,
+            serial_number=test.serial_number,
             result=result,
             note=note,
             duration=duration,

tmt/steps/discover/__init__.py

@@ -279,6 +279,9 @@ def go(self) -> None:
             else:
                 raise GeneralError(f'Unexpected phase in discover step: {phase}')
 
+        for test in self._tests:
+            test.serial_number = self.plan.draw_test_serial_number(test)
+
         # Show fmf identifiers for tests discovered in plan
         # TODO: This part should go into the 'fmf.py' module
         if self.opt('fmf_id'):

tmt/steps/execute/__init__.py

@@ -200,8 +200,10 @@ def data_path(
         filename not provided) or to the given data file otherwise.
         """
         # Prepare directory path, create if requested
-        assert self.step.workdir is not None
-        directory = self.step.workdir / TEST_DATA / test.name.lstrip('/')
+        assert self.step.workdir is not None  # narrow type
+        directory = self.step.workdir \
+            / TEST_DATA \
+            / f'{test.name.lstrip("/")}-{test.serial_number}'
         if create and not directory.is_dir():
             directory.joinpath(TEST_DATA).mkdir(parents=True)
         if not filename: