Skip to content

Commit

Permalink
Simplify the Result class implementation
Browse files Browse the repository at this point in the history
* drop `ResultData`, it's no longer needed and `Result` can handle all
  the work;
* add `guest` key as proposed in [1];
* add schema for validation of (custom) `results.yaml` (the schema is
  not applied yet);
* `execute` step saves `results.yaml` as a list of results rather than a
  mapping, to make allow one test being executed multiple times. See [2].

[1] #1614 (comment)
[2] #1614
  • Loading branch information
happz committed Feb 22, 2023
1 parent 1dc5bf8 commit 3e2b696
Show file tree
Hide file tree
Showing 8 changed files with 278 additions and 208 deletions.
50 changes: 23 additions & 27 deletions tests/execute/basic/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,10 @@
rlJournalStart
function check_duration() {
local result_file=$1
local results=$2
DURATION_REGEXP="^(((([0-1][0-9])|(2[0-3])):?[0-5][0-9]:?[0-5][0-9]+$))"
DURATION=$(grep -A5 "$results" "$result_file" | grep "duration:" | awk '{print $2}')
if [[ "$DURATION" =~ $DURATION_REGEXP ]]; then
rlRun "true" 0 "duration is in HH:MM:SS format"
else
rlRun "false" 0 "duration isn't in HH:MM:SS format"
fi
local test_name=$2

rlRun "yq -ery '.[] | select(.name == \"$test_name\") | .duration | test(\"^[0-9]{2,}:[0-5][0-9]:[0-5][0-9]$\")' $result_file" \
0 "duration is in HH:MM:SS format"
}

rlPhaseStartSetup
Expand All @@ -38,40 +34,40 @@ rlJournalStart
rlPhaseStartTest "Check shell results"
results="$run/plan/shell/execute/results.yaml"

rlRun "grep -Pzo '(?sm)^/test/shell/good:$.*?^ *result: pass$' $results" 0 "Check pass"
check_duration "$results" "good:"
rlRun "yq -ery '.[] | select(.name == \"/test/shell/good\" and .result == \"pass\")' $results" 0 "Check pass"
check_duration "$results" "/test/shell/good"

rlRun "grep -Pzo '(?sm)^/test/shell/weird:$.*?^ *result: error$' $results" 0 "Check error"
check_duration "$results" "weird:"
rlRun "yq -ery '.[] | select(.name == \"/test/shell/weird\" and .result == \"error\")' $results" 0 "Check error"
check_duration "$results" "/test/shell/weird"

rlRun "grep -Pzo '(?sm)^/test/shell/bad:$.*?^ *result: fail$' $results" 0 "Check fail"
check_duration "$results" "bad:"
rlRun "yq -ery '.[] | select(.name == \"/test/shell/bad\" and .result == \"fail\")' $results" 0 "Check fail"
check_duration "$results" "/test/shell/bad"

# Check log file exists
rlRun "grep -Pzo '(?sm)^/test/shell/good:$.*?^ +log:$.*?^ +- data/.+?$' $results | grep output.txt" \
rlRun "yq -ery '.[] | select(.name == \"/test/shell/good\") | .log | .[] | test(\"^data/.+/output.txt$\")' $results" \
0 "Check output.txt log exists in $results"
rlPhaseEnd

rlPhaseStartTest "Check beakerlib results"
results="$run/plan/beakerlib/execute/results.yaml"

rlRun "grep -Pzo '(?sm)^/test/beakerlib/good:$.*?^ *result: pass$' $results" 0 "Check pass"
check_duration "$results" "good:"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/good\" and .result == \"pass\")' $results" 0 "Check pass"
check_duration "$results" "/test/beakerlib/good"

rlRun "grep -Pzo '(?sm)^/test/beakerlib/need:$.*?^ *result: warn$' $results" 0 "Check warn"
check_duration "$results" "need:"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/need\" and .result == \"warn\")' $results" 0 "Check warn"
check_duration "$results" "/test/beakerlib/need"

rlRun "grep -Pzo '(?sm)^/test/beakerlib/weird:$.*?^ *result: error$' $results" 0 "Check error"
check_duration "$results" "weird:"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/weird\" and .result == \"error\")' $results" 0 "Check error"
check_duration "$results" "/test/beakerlib/weird"

rlRun "grep -Pzo '(?sm)^/test/beakerlib/bad:$.*?^ *result: fail$' $results" 0 "Check fail"
check_duration "$results" "bad:"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/bad\" and .result == \"fail\")' $results" 0 "Check fail"
check_duration "$results" "/test/beakerlib/bad"

# Check log files exist
rlRun "grep -Pzo '(?sm)^/test/beakerlib/good:$.*^ +log:$.*?^ +- data/.+?$' $results | grep output.txt" \
0 "Check output.txt log exists"
rlRun "grep -Pzo '(?sm)^/test/beakerlib/good:$.*^ +log:$.*?^ +- data/.+?$' $results | grep journal.txt" \
0 "Check journal.txt log exists"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/good\") | .log | map({path: .}) | .[] | select(.path | test(\"^data/.+/output.txt$\"))' $results" \
0 "Check output.txt log exists in $results"
rlRun "yq -ery '.[] | select(.name == \"/test/beakerlib/good\") | .log | map({path: .}) | .[] | select(.path | test(\"^data/.+/journal.txt$\"))' $results" \
0 "Check journal.txt log exists in $results"
rlPhaseEnd

rlPhaseStartCleanup
Expand Down
2 changes: 1 addition & 1 deletion tests/execute/result/custom/wrong_results.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# TODO: Make also this test working - JSON scheme needs to be defined for
# ResultData to prevent invalid key 'foo'
# Result to prevent invalid key 'foo'
#
#- name: /test/wrong-key
# result: pass
Expand Down
42 changes: 6 additions & 36 deletions tests/unit/test_report_junit.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import pytest

from tmt.result import Result, ResultData, ResultOutcome
from tmt.result import Result, ResultOutcome
from tmt.steps.report.junit import ReportJUnit, ReportJUnitData
from tmt.utils import Path

Expand Down Expand Up @@ -128,13 +128,7 @@ def assert_xml(actual_filepath, expected):
class TestStateMapping:
def test_pass(self, report_fix):
report, results, out_file_path = report_fix
results.extend(
[
Result(
data=ResultData(result=ResultOutcome.PASS),
name="/pass")
]
)
results.append(Result(result=ResultOutcome.PASS, name="/pass"))

report.go()

Expand All @@ -148,13 +142,7 @@ def test_pass(self, report_fix):

def test_info(self, report_fix):
report, results, out_file_path = report_fix
results.extend(
[
Result(
data=ResultData(result=ResultOutcome.INFO),
name="/info")
]
)
results.append(Result(result=ResultOutcome.INFO, name="/info"))
report.go()

assert_xml(out_file_path, """<?xml version="1.0" ?>
Expand All @@ -169,13 +157,7 @@ def test_info(self, report_fix):

def test_warn(self, report_fix):
report, results, out_file_path = report_fix
results.extend(
[
Result(
data=ResultData(result=ResultOutcome.WARN),
name="/warn")
]
)
results.append(Result(result=ResultOutcome.WARN, name="/warn"))
report.go()

assert_xml(out_file_path, """<?xml version="1.0" ?>
Expand All @@ -190,13 +172,7 @@ def test_warn(self, report_fix):

def test_error(self, report_fix):
report, results, out_file_path = report_fix
results.extend(
[
Result(
data=ResultData(result=ResultOutcome.ERROR),
name="/error")
]
)
results.append(Result(result=ResultOutcome.ERROR, name="/error"))
report.go()

assert_xml(out_file_path, """<?xml version="1.0" ?>
Expand All @@ -211,13 +187,7 @@ def test_error(self, report_fix):

def test_fail(self, report_fix):
report, results, out_file_path = report_fix
results.extend(
[
Result(
data=ResultData(result=ResultOutcome.FAIL),
name="/fail")
]
)
results.append(Result(result=ResultOutcome.FAIL, name="/fail"))
report.go()

assert_xml(out_file_path, """<?xml version="1.0" ?>
Expand Down
Loading

0 comments on commit 3e2b696

Please sign in to comment.