Assign a data path and serial number to each test in discover

spec/plans/discover.fmf

@@ -30,6 +30,7 @@ description: |
             result: respect
             tag: [tag]
             tier: 1
+            serialnumber: 1
 
           - name: /test/two
             summary: Short test summary.

tests/discover/main.fmf

@@ -56,3 +56,7 @@
 /force:
     summary: Force run of discover finds tests
     test: ./force.sh
+
+/serial-number:
+    summary: Test test serial number assignment
+    test: ./serial-number.sh

tests/discover/order.sh

@@ -6,14 +6,14 @@
 function assert_execution_order(){
     local input_file="$tmp/run/plans${plan}/data/execution_order"
     sed -n 's;^.*execute/data\(.\+\)/data;\1;p' "$input_file" > $tmp/outcome
-    rlRun "diff -pu $tmp/outcome $tmp/EXPECTED"
+    rlRun "diff -pu $tmp/outcome $tmp/EXPECTED-EXECUTION" 0 "Verify execution order"
 }
 
 function assert_discover_order(){
     local input_file="$rlRun_LOG"
     # Discovered tests are printed with leading spaces
     sed -n 's;^ \+\(/.\+\);\1;p' "$input_file" > $tmp/outcome
-    rlRun "diff -pu $tmp/outcome $tmp/EXPECTED"
+    rlRun "diff -pu $tmp/outcome $tmp/EXPECTED-DISCOVERY" 0 "Verify discovery order"
 }
 
 function run_test(){
@@ -43,55 +43,83 @@ rlJournalStart
 
     ### New test begins
     plan="/single-without-order-tag"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /tests/no-order-0
 /tests/no-order-1
 /tests/no-order-2
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/tests/no-order-0-1
+/tests/no-order-1-2
+/tests/no-order-2-3
 EOF
     run_test
 
     ### New test begins
     plan="/single-without-order-name"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /tests/no-order-0
 /tests/no-order-1
 /tests/no-order-2
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/tests/no-order-0-1
+/tests/no-order-1-2
+/tests/no-order-2-3
 EOF
     run_test
 
     ### New test begins
     plan="/single-enumerate"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /tests/no-order-2
 /tests/no-order-0
 /tests/no-order-1
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/tests/no-order-2-1
+/tests/no-order-0-2
+/tests/no-order-1-3
 EOF
     run_test
 
     ### New test begins
     plan="/single-enumerate-and-order"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /tests/no-order-2
 /tests/order-80
 /tests/no-order-0
 /tests/no-order-1
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/tests/no-order-2-1
+/tests/order-80-2
+/tests/no-order-0-3
+/tests/no-order-1-4
 EOF
     run_test
 
     ### New test begins
     plan="/single-order"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /tests/order-10
 /tests/no-order-0
 /tests/no-order-1
 /tests/no-order-2
 /tests/order-80
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/tests/order-10-1
+/tests/no-order-0-2
+/tests/no-order-1-3
+/tests/no-order-2-4
+/tests/order-80-5
 EOF
     run_test
 
     ### New test begins
     plan="/multiple-by-enumerate"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /enumerate-and-order/tests/no-order-2
 /enumerate-and-order/tests/order-80
 /enumerate-and-order/tests/no-order-0
@@ -103,12 +131,25 @@ EOF
 /by-order-attribute/tests/order-80
 /third/order-20
 /third/order-default
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/enumerate-and-order/tests/no-order-2-1
+/enumerate-and-order/tests/order-80-2
+/enumerate-and-order/tests/no-order-0-3
+/enumerate-and-order/tests/no-order-1-4
+/by-order-attribute/tests/order-10-5
+/by-order-attribute/tests/no-order-0-6
+/by-order-attribute/tests/no-order-1-7
+/by-order-attribute/tests/no-order-2-8
+/by-order-attribute/tests/order-80-9
+/third/order-20-10
+/third/order-default-11
 EOF
     run_test
 
     ### New test begins
     plan="/multiple-by-order"
-    cat > $tmp/EXPECTED <<EOF
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
 /order-10/tests/order-10
 /order-10/tests/no-order-0
 /order-10/tests/no-order-1
@@ -120,6 +161,19 @@ EOF
 /order-80/tests/order-80
 /order-80/tests/no-order-0
 /order-80/tests/no-order-1
+EOF
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/order-10/tests/order-10-1
+/order-10/tests/no-order-0-2
+/order-10/tests/no-order-1-3
+/order-10/tests/no-order-2-4
+/order-10/tests/order-80-5
+/order-default/order-20-6
+/order-default/order-default-7
+/order-80/tests/no-order-2-8
+/order-80/tests/order-80-9
+/order-80/tests/no-order-0-10
+/order-80/tests/no-order-1-11
 EOF
     run_test
 

tests/discover/serial-number-root-test/.fmf/version

@@ -0,0 +1 @@
+1

tests/discover/serial-number-root-test/main.fmf

@@ -0,0 +1 @@
+test: /bin/true

tests/discover/serial-number.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "workdir=\$(mktemp -d)" 0 "Create tmp directory"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Single discover phase"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/single-discover'"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/single-discover/discover/tests.yaml"
+        rlAssertGrep "/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/tests/foo 2" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Multiple discover phases"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/multiple-discover'"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/multiple-discover/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+        rlAssertGrep "/default-2/tests/bar 5" $rlRun_LOG
+        rlAssertGrep "/default-2/tests/foo 6" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Multiple plans"
+        rlRun 'pushd serial-number'
+        rlRun "tmt -vv run --scratch --id $workdir discover plan --name '/multiple-plans'"
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/multiple-plans/plan1/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/multiple-plans/plan2/discover/tests.yaml"
+        rlAssertGrep "/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/tests/foo 2" $rlRun_LOG
+
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/multiple-plans/plan3/discover/tests.yaml"
+        rlAssertGrep "/default-0/tests/bar 1" $rlRun_LOG
+        rlAssertGrep "/default-0/tests/foo 2" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/bar 3" $rlRun_LOG
+        rlAssertGrep "/default-1/tests/foo 4" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartTest "Single '/' test"
+        rlRun 'pushd serial-number-root-test'
+        rlRun "tmt -vv run --scratch --id $workdir discover -h fmf"
+        rlRun -s "yq -er '.[] | \"\\(.name) \\(.serialnumber)\"' $workdir/plans/default/discover/tests.yaml"
+        rlAssertGrep "/ 1" $rlRun_LOG
+        rlRun 'popd'
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        # rlRun "rm -rf $workdir" 0 'Remove tmp directory'
+    rlPhaseEnd
+rlJournalEnd

tests/discover/serial-number/.fmf/version

@@ -0,0 +1 @@
+1

tests/discover/serial-number/plans.fmf

@@ -0,0 +1,29 @@
+execute:
+    how: tmt
+provision:
+    how: local
+
+/single-discover:
+    discover:
+        how: fmf
+
+/multiple-discover:
+    discover:
+        - how: fmf
+        - how: fmf
+        - how: fmf
+
+/multiple-plans:
+    /plan1:
+        discover:
+          - how: fmf
+          - how: fmf
+
+    /plan2:
+        discover:
+          how: fmf
+
+    /plan3:
+        discover:
+          - how: fmf
+          - how: fmf

tests/discover/serial-number/tests.fmf

@@ -0,0 +1,5 @@
+/foo:
+  test: /bin/true
+
+/bar:
+  test: /bin/true

tests/execute/filesubmit/test.sh

@@ -9,7 +9,7 @@ rlJournalStart
     rlPhaseEnd
 
     # would be set by TMT_TEST_DATA
-    tmt_test_data="plans/default/execute/data/data"
+    tmt_test_data="plans/default/execute/data/default-1/data"
 
     rlPhaseStartTest
         rlRun "tmt run -vfi $tmp -a provision -h container"

tests/execute/metadata/test.sh

@@ -10,7 +10,7 @@ rlJournalStart
 
     rlPhaseStartTest
         rlRun "tmt run -vi $tmp"
-        metadata="$tmp/plan/execute/data/test/metadata.yaml"
+        metadata="$tmp/plan/execute/data/test-1/metadata.yaml"
         rlRun "cat $metadata" 0 "Check metadata.yaml content"
         rlAssertGrep "name: /test" $metadata
         rlAssertGrep "summary: Simple test" $metadata

tests/execute/reboot/basic.sh

@@ -22,7 +22,7 @@ rlJournalStart
             # Check that the whole output log is kept
             # The test output is not stored in log in interactive mode
             if [ -z "$interactive" ]; then
-                rlRun "log=$run/plan/execute/data/test/output.txt"
+                rlRun "log=$run/plan/execute/data/test-1/output.txt"
                 rlAssertGrep "After first reboot" $log
                 rlAssertGrep "After second reboot" $log
                 rlAssertGrep "After third reboot" $log

tests/execute/result/custom.sh

@@ -26,7 +26,7 @@ rlJournalStart
     testName="/test/missing-custom-results"
     rlPhaseStartTest "${testName}"
         rlRun -s "${tmt_command} ${testName} 2>&1 >/dev/null" 2 "Test does not provide 'results.yaml' file"
-        rlAssertGrep "custom results file '/tmp/.*/plans/default/execute/data/test/missing-custom-results/data/results.yaml' not found" $rlRun_LOG
+        rlAssertGrep "custom results file '/tmp/.*/plans/default/execute/data/test/missing-custom-results-1/data/results.yaml' not found" $rlRun_LOG
     rlPhaseEnd
 
     testName="/test/empty-custom-results-file"

tests/execute/upgrade/ignore-test.sh

@@ -20,10 +20,10 @@ rlJournalStart
         rlAssertGrep "3 tests passed" $rlRun_LOG
         # Check that the IN_PLACE_UPGRADE variable was set
         data="$run/plan/no-path/execute/data"
-        rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test/output.txt"
-        rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test/output.txt"
+        rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test-1/output.txt"
+        rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test-1/output.txt"
         # No upgrade path -> no environment variable
-        rlAssertNotGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare/output.txt"
+        rlAssertNotGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare-0/output.txt"
     rlPhaseEnd
 
     rlPhaseStartCleanup

tests/execute/upgrade/override.sh

@@ -23,10 +23,10 @@ rlJournalStart
             rlAssertGrep "3 tests passed" $rlRun_LOG
             # Check that the IN_PLACE_UPGRADE variable was set
             data="$run/plan/path/execute/data"
-            rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test/output.txt"
-            rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test/output.txt"
+            rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test-1/output.txt"
+            rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test-1/output.txt"
             # Environment of plan was passed
-            rlAssertGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare/output.txt"
+            rlAssertGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare-0/output.txt"
         rlPhaseEnd
     done
 

tests/execute/upgrade/simple.sh

@@ -20,10 +20,10 @@ rlJournalStart
         rlAssertGrep "3 tests passed" $rlRun_LOG
         # Check that the IN_PLACE_UPGRADE variable was set
         data="$run/plan/no-path/execute/data"
-        rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test/output.txt"
-        rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test/output.txt"
+        rlAssertGrep "IN_PLACE_UPGRADE=old" "$data/old/test-1/output.txt"
+        rlAssertGrep "IN_PLACE_UPGRADE=new" "$data/new/test-1/output.txt"
         # No upgrade path -> no environment variable
-        rlAssertNotGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare/output.txt"
+        rlAssertNotGrep "VERSION_ID=$PREVIOUS_VERSION" "$data/upgrade/tasks/prepare-0/output.txt"
     rlPhaseEnd
 
     rlPhaseStartCleanup

tests/finish/prune/test.sh

@@ -14,7 +14,7 @@ rlJournalStart
         rlAssertNotExists $tmp1/plan/discover/default-0
         rlAssertNotExists $tmp1/plan/discover/default-1
         rlAssertExists $tmp1/plan/data/out-plan.txt
-        rlAssertExists $tmp1/plan/execute/data/default-2/write/test-data/data/out-test.txt
+        rlAssertExists $tmp1/plan/execute/data/default-2/write/test-data-3/data/out-test.txt
         for step in discover execute finish prepare provision report; do
             rlAssertExists $tmp1/plan/$step/step.yaml
         done
@@ -37,7 +37,7 @@ rlJournalStart
         rlAssertExists $tmp2/plan/discover/default-0
         rlAssertExists $tmp2/plan/discover/default-1
         rlAssertExists $tmp2/plan/data/out-plan.txt
-        rlAssertExists $tmp2/plan/execute/data/default-2/write/test-data/data/out-test.txt
+        rlAssertExists $tmp2/plan/execute/data/default-2/write/test-data-3/data/out-test.txt
         for step in discover execute finish prepare provision report; do
             rlAssertExists $tmp2/plan/$step/step.yaml
         done