From d2680fe44afb1d5242082fba7766fc4d519cffb2 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Mon, 1 Aug 2022 17:31:18 +0200 Subject: [PATCH] feat: detailed steps output (#487) * feat: detailed steps output Signed-off-by: GitHub --- README.md | 2 + assertion.go | 30 +++++------ process_testcase.go | 92 +++++++++++++++++++++++++++----- process_teststep.go | 42 +++++++++------ process_testsuite.go | 35 +++++++----- tests/failing/verbose_output.yml | 54 +++++++++++++++++++ tests/verbose_output.yml | 35 ++++++++++++ types.go | 16 ++++-- types_executor.go | 4 +- venom.go | 6 ++- 10 files changed, 256 insertions(+), 60 deletions(-) create mode 100644 tests/failing/verbose_output.yml create mode 100644 tests/verbose_output.yml diff --git a/README.md b/README.md index a8718f00..f1730771 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,8 @@ Flags and their equivalent with environment variables usage: - `-v` flag is equivalent to `VENOM_VERBOSE=1` environment variable - `-vv` flag is equivalent to `VENOM_VERBOSE=2` environment variable +It is possible to set `NO_COLOR=1` environment variable to disable colors from output. + ## Use a configuration file You can define the Venom settings using a configuration file `.venomrc`. This configuration file should be placed in the current directory or in the `home` directory. diff --git a/assertion.go b/assertion.go index 663a0ab8..cfb0a3c2 100644 --- a/assertion.go +++ b/assertion.go @@ -24,7 +24,7 @@ type assertionsApplied struct { systemerr string } -func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep, defaultAssertions *StepAssertions) assertionsApplied { +func applyAssertions(r interface{}, tc TestCase, stepNumber int, rangedIndex int, step TestStep, defaultAssertions *StepAssertions) assertionsApplied { var sa StepAssertions var errors []Failure var failures []Failure @@ -48,7 +48,7 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep, isOK := true for _, assertion := range sa.Assertions { - errs, fails := check(tc, stepNumber, assertion, executorResult) + errs, fails := check(tc, stepNumber, rangedIndex, assertion, executorResult) if errs != nil { errors = append(errors, *errs) isOK = false @@ -116,26 +116,26 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti } // check selects the correct assertion function to call depending on typing provided by user -func check(tc TestCase, stepNumber int, assertion Assertion, r interface{}) (*Failure, *Failure) { +func check(tc TestCase, stepNumber int, rangedIndex int, assertion Assertion, r interface{}) (*Failure, *Failure) { var errs *Failure var fails *Failure switch t := assertion.(type) { case string: - errs, fails = checkString(tc, stepNumber, assertion.(string), r) + errs, fails = checkString(tc, stepNumber, rangedIndex, assertion.(string), r) case map[string]interface{}: - errs, fails = checkBranch(tc, stepNumber, assertion.(map[string]interface{}), r) + errs, fails = checkBranch(tc, stepNumber, rangedIndex, assertion.(map[string]interface{}), r) default: - errs = newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion format: %v", t)) + errs = newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("unsupported assertion format: %v", t)) } return errs, fails } // checkString evaluate a complex assertion containing logical operators // it recursively calls checkAssertion for each operand -func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r interface{}) (*Failure, *Failure) { +func checkBranch(tc TestCase, stepNumber int, rangedIndex int, branch map[string]interface{}, r interface{}) (*Failure, *Failure) { // Extract logical operator if len(branch) != 1 { - return newFailure(tc, stepNumber, "", fmt.Errorf("expected exactly 1 logical operator but %d were provided", len(branch))), nil + return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("expected exactly 1 logical operator but %d were provided", len(branch))), nil } var operator string for k := range branch { @@ -148,7 +148,7 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i case []interface{}: operands = branch[operator].([]interface{}) default: - return newFailure(tc, stepNumber, "", fmt.Errorf("expected %s operands to be an []interface{}, got %v", operator, t)), nil + return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("expected %s operands to be an []interface{}, got %v", operator, t)), nil } if len(operands) == 0 { return nil, nil @@ -161,7 +161,7 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i assertionsCount := len(operands) assertionsSuccess := 0 for _, assertion := range operands { - errs, fails := check(tc, stepNumber, assertion, r) + errs, fails := check(tc, stepNumber, rangedIndex, assertion, r) if errs != nil { errsBuf = append(errsBuf, *errs) } @@ -198,23 +198,23 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i err = fmt.Errorf("some assertions succeeded but expected none to suceed:\n%s\n", strings.Join(results, "\n")) } default: - return newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion operator %s", operator)), nil + return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("unsupported assertion operator %s", operator)), nil } if err != nil { - return nil, newFailure(tc, stepNumber, "", err) + return nil, newFailure(tc, stepNumber, rangedIndex, "", err) } return nil, nil } // checkString evaluate a single string assertion -func checkString(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) { +func checkString(tc TestCase, stepNumber int, rangedIndex int, assertion string, r interface{}) (*Failure, *Failure) { assert, err := parseAssertions(context.Background(), assertion, r) if err != nil { - return nil, newFailure(tc, stepNumber, assertion, err) + return nil, newFailure(tc, stepNumber, rangedIndex, assertion, err) } if err := assert.Func(assert.Actual, assert.Args...); err != nil { - failure := newFailure(tc, stepNumber, assertion, err) + failure := newFailure(tc, stepNumber, rangedIndex, assertion, err) failure.AssertionRequired = assert.Required return nil, failure } diff --git a/process_testcase.go b/process_testcase.go index 2a2ba818..35fdedd5 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" + "github.com/fatih/color" "github.com/ghodss/yaml" "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" @@ -146,10 +147,11 @@ func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) { } defer Info(ctx, "Ending testcase") - v.runTestSteps(ctx, tc) + v.runTestSteps(ctx, tc, nil) } -func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { +func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepResult) { + results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v") if err != nil { Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) @@ -166,6 +168,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { var knowExecutors = map[string]struct{}{} var previousStepVars = H{} + fromUserExecutor := tsIn != nil for stepNumber, rawStep := range tc.RawTestSteps { stepVars := tc.Vars.Clone() @@ -181,8 +184,10 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { } for rangedIndex, rangedData := range ranged.Items { + tc.TestStepResults = append(tc.TestStepResults, TestStepResult{}) + ts := &tc.TestStepResults[len(tc.TestStepResults)-1] if ranged.Enabled { - Debug(ctx, "processing step %d", rangedIndex) + Debug(ctx, "processing range index: %d", rangedIndex) stepVars.Add("index", rangedIndex) stepVars.Add("key", rangedData.Key) stepVars.Add("value", rangedData.Value) @@ -225,7 +230,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { } } - Info(ctx, "Step #%d content is: %q", stepNumber, content) + Info(ctx, "Step #%d-%d content is: %q", stepNumber, rangedIndex, content) var step TestStep if err := yaml.Unmarshal([]byte(content), &step); err != nil { tc.AppendError(err) @@ -261,7 +266,10 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { } } - result := v.RunTestStep(ctx, e, tc, stepNumber, step) + printStepName := v.Verbose >= 2 && !fromUserExecutor + v.setTestStepName(ts, e, step, &ranged, &rangedData, printStepName) + + result := v.RunTestStep(ctx, e, tc, ts, stepNumber, rangedIndex, step) mapResult := GetExecutorResult(result) previousStepVars.AddAll(H(mapResult)) @@ -269,17 +277,17 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { var hasFailed bool var isRequired bool - if len(tc.Failures) > 0 { - for _, f := range tc.Failures { + if len(ts.Failures) > 0 { + for _, f := range ts.Failures { Warning(ctx, "%v", f) isRequired = isRequired || f.AssertionRequired } hasFailed = true } - if len(tc.Errors) > 0 { + if len(ts.Errors) > 0 { Error(ctx, "Errors: ") - for _, e := range tc.Errors { + for _, e := range ts.Errors { Error(ctx, "%v", e) } hasFailed = true @@ -287,12 +295,15 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { if hasFailed { if isRequired { - failure := newFailure(*tc, stepNumber, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) - tc.Failures = append(tc.Failures, *failure) + failure := newFailure(*tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) + ts.appendFailure(tc, *failure) + v.printTestStepResult(tc, ts, tsIn, stepNumber, true) return } - break + v.printTestStepResult(tc, ts, tsIn, stepNumber, false) + continue } + v.printTestStepResult(tc, ts, tsIn, stepNumber, false) allVars := tc.Vars.Clone() allVars.AddAll(tc.computedVars.Clone()) @@ -310,6 +321,63 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { } } +//Set test step name (defaults to executor name, excepted if it got a "name" attribute. in range, also print key) +func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestStep, ranged *Range, rangedData *RangeData, print bool) { + name := e.Name() + if value, ok := step["name"]; ok { + switch value := value.(type) { + case string: + name = value + } + } + if ranged.Enabled { + name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key) + } + ts.Name = name + + if print { + v.Print(" \t\t• %s", ts.Name) + } +} + +//Print a single step result (if verbosity is enabled) +func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) { + if v.Verbose >= 2 { + fromUserExecutor := tsIn != nil + var red = color.New(color.FgRed).SprintFunc() + var green = color.New(color.FgGreen).SprintFunc() + var gray = color.New(color.Attribute(90)).SprintFunc() + + //Within an user executor, we instead transfer errors to original test step which will print it for use + if fromUserExecutor { + tsIn.appendError(tc, ts.Errors...) + tsIn.appendFailure(tc, ts.Failures...) + } else { //Else print step status + if len(ts.Skipped) > 0 { + v.Println(" %s", gray("SKIPPED")) + } else if len(ts.Failures) > 0 || len(ts.Errors) > 0 { + v.Println(" %s", red("FAILURE")) + for _, f := range ts.Failures { + v.Println(" \t\t %s", red(f)) + } + for _, f := range ts.Errors { + v.Println(" \t\t %s", red(f.Value)) + } + if mustAssertionFailed { + skipped := len(tc.RawTestSteps) - stepNumber - 1 + if skipped == 1 { + v.Println(" \t\t %s", gray(fmt.Sprintf("%d other step was skipped", skipped))) + } else { + v.Println(" \t\t %s", gray(fmt.Sprintf("%d other steps were skipped", skipped))) + } + } + } else { + v.Println(" %s", green("SUCCESS")) + } + } + } +} + //Parse and format range data to allow iterations over user data func parseRanged(ctx context.Context, rawStep []byte, stepVars H) (Range, error) { diff --git a/process_teststep.go b/process_teststep.go index cb3ec16f..3c16f832 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -19,7 +19,7 @@ type dumpFile struct { } //RunTestStep executes a venom testcase is a venom context -func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, stepNumber int, step TestStep) interface{} { +func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) var assertRes assertionsApplied @@ -33,12 +33,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } var err error - result, err = v.runTestStepExecutor(ctx, e, tc, step) + result, err = v.runTestStepExecutor(ctx, e, tc, ts, step) if err != nil { // we save the failure only if it's the last attempt if retry == e.Retry() { - failure := newFailure(*tc, stepNumber, "", err) - tc.Failures = append(tc.Failures, *failure) + failure := newFailure(*tc, stepNumber, rangedIndex, "", err) + ts.appendFailure(tc, *failure) } continue } @@ -62,7 +62,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, if oDir == "" { oDir = "." } - filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber)) + filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex)) if err := os.WriteFile(filename, []byte(output), 0644); err != nil { return fmt.Errorf("Error while creating file %s: %v", filename, err) @@ -97,12 +97,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, if result == nil { Debug(ctx, "empty testcase, applying assertions on variables: %v", AllVarsFromCtx(ctx)) - assertRes = applyAssertions(AllVarsFromCtx(ctx), *tc, stepNumber, step, nil) + assertRes = applyAssertions(AllVarsFromCtx(ctx), *tc, stepNumber, rangedIndex, step, nil) } else { if h, ok := e.(executorWithDefaultAssertions); ok { - assertRes = applyAssertions(result, *tc, stepNumber, step, h.GetDefaultAssertions()) + assertRes = applyAssertions(result, *tc, stepNumber, rangedIndex, step, h.GetDefaultAssertions()) } else { - assertRes = applyAssertions(result, *tc, stepNumber, step, nil) + assertRes = applyAssertions(result, *tc, stepNumber, rangedIndex, step, nil) } } @@ -116,16 +116,16 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, return fmt.Errorf("Error while evaluating retry condition: %v", err) } if len(failures) > 0 { - failure := newFailure(*tc, stepNumber, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-retry)) + failure := newFailure(*tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-retry)) tc.Failures = append(tc.Failures, *failure) break } } - tc.Errors = append(tc.Errors, assertRes.errors...) - tc.Failures = append(tc.Failures, assertRes.failures...) + ts.appendError(tc, assertRes.errors...) + ts.appendFailure(tc, assertRes.failures...) if retry > 1 && (len(assertRes.failures) > 0 || len(assertRes.errors) > 0) { - tc.Failures = append(tc.Failures, Failure{Value: fmt.Sprintf("It's a failure after %d attempts", retry)}) + ts.appendFailure(tc, Failure{Value: fmt.Sprintf("It's a failure after %d attempts", retry)}) } tc.Systemout.Value += assertRes.systemout tc.Systemerr.Value += assertRes.systemerr @@ -133,12 +133,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, return result } -func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, step TestStep) (interface{}, error) { +func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) if e.Timeout() == 0 { if e.Type() == "user" { - return v.RunUserExecutor(ctx, e, tc, step) + return v.RunUserExecutor(ctx, e, tc, ts, step) } return e.Run(ctx, step) } @@ -152,7 +152,7 @@ func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *T var err error var result interface{} if e.Type() == "user" { - result, err = v.RunUserExecutor(ctx, e, tc, step) + result, err = v.RunUserExecutor(ctx, e, tc, ts, step) } else { result, err = e.Run(ctx, step) } @@ -172,3 +172,15 @@ func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *T return nil, fmt.Errorf("Timeout after %d second(s)", e.Timeout()) } } + +//Append an error to a test step and its associated test case +func (ts *TestStepResult) appendError(tc *TestCase, failure ...Failure) { + ts.Errors = append(ts.Errors, failure...) + tc.Errors = append(tc.Errors, failure...) +} + +//Append an assertion failure to a test step and its associated test case +func (ts *TestStepResult) appendFailure(tc *TestCase, failure ...Failure) { + ts.Failures = append(ts.Failures, failure...) + tc.Failures = append(tc.Failures, failure...) +} diff --git a/process_testsuite.go b/process_testsuite.go index 911809f3..7b2cb891 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -72,6 +72,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { var green = color.New(color.FgGreen).SprintFunc() var cyan = color.New(color.FgCyan).SprintFunc() var gray = color.New(color.Attribute(90)).SprintFunc() + verboseReport := v.Verbose >= 2 v.Println(" • %s (%s)", ts.Name, ts.Package) @@ -79,6 +80,9 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { tc := &ts.TestCases[i] tc.IsEvaluated = true v.Print(" \t• %s", tc.Name) + if verboseReport { + v.Print("\n") + } tc.Classname = ts.Filename var hasFailure bool var hasSkipped = len(tc.Skipped) > 0 @@ -101,28 +105,35 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { hasSkipped = true } - if hasSkipped { - v.Println(" %s", gray("SKIPPED")) - continue - } - - if hasFailure { - v.Println(" %s", red("FAILURE")) + // Verbose mode already reported tests status, so just print them when non-verbose + indent := "" + if verboseReport { + indent = "\t " } else { - v.Println(" %s", green("SUCCESS")) + if hasSkipped { + v.Println(" %s", gray("SKIPPED")) + continue + } + + if hasFailure { + v.Println(" %s", red("FAILURE")) + } else { + v.Println(" %s", green("SUCCESS")) + } } for _, i := range tc.computedInfo { - v.Println("\t %s %s", cyan("[info]"), cyan(i)) + v.Println("\t %s%s %s", indent, cyan("[info]"), cyan(i)) } for _, i := range tc.computedVerbose { - v.PrintlnTrace(i) + v.PrintlnIndentedTrace(i, indent) } - if hasFailure { + // Verbose mode already reported failures, so just print them when non-verbose + if !verboseReport && hasFailure { for _, f := range tc.Failures { - v.Println("%s", red(f.Value)) + v.Println("%s", red(f)) } for _, f := range tc.Errors { v.Println("%s", red(f.Value)) diff --git a/tests/failing/verbose_output.yml b/tests/failing/verbose_output.yml new file mode 100644 index 00000000..271ce468 --- /dev/null +++ b/tests/failing/verbose_output.yml @@ -0,0 +1,54 @@ +name: Test detailed output +testcases: +- name: Test single step + steps: + - type: exec + script: echo foo + assertions: + - result.systemout ShouldEqual foo +- name: Test named step + steps: + - name: hello-world + type: exec + script: echo foo + assertions: + - result.systemout ShouldEqual foo +- name: Test multi step + steps: + - name: step1 + type: exec + script: echo foo + assertions: + - result.systemout ShouldEqual foo + - name: step2 + type: exec + script: echo bar + assertions: + - result.systemout ShouldEqual foo + - result.systemout ShouldEqual baz +- name: Test ranged steps + steps: + - type: exec + script: echo {{.value.v}} + assertions: + - result.systemout ShouldEqual foo + range: + - k: a + v: foo + - k: b + v: bar + - k: c + v: foo +- name: Test must assertions + steps: + - name: must1 + type: exec + script: echo foo + assertions: + - result.systemout MustEqual bar + - name: must2 + type: exec + script: echo foo + - name: must3 + type: exec + script: echo bar \ No newline at end of file diff --git a/tests/verbose_output.yml b/tests/verbose_output.yml new file mode 100644 index 00000000..476b30c7 --- /dev/null +++ b/tests/verbose_output.yml @@ -0,0 +1,35 @@ +name: testsuite run in verbose mode +testcases: +- name: testsuite run in verbose mode + steps: + # spawn a venom sub-process and expect it to fail and make assertions on its error messages + # ensure no color to avoid annoying checks + - type: exec + script: NO_COLOR=1 {{.venom.executable}} run failing/verbose_output.yml {{.value.opt}} + range: + verbose: + opt: "-vv" + op: Should + default: + opt: "" + op: ShouldNot + assertions: + - result.code ShouldEqual 2 + - result.systemerr ShouldBeEmpty + # single step + - result.systemout {{.value.op}}ContainSubstring 'exec SUCCESS' + # named step + - result.systemout {{.value.op}}ContainSubstring 'hello-world SUCCESS' + # multi steps + - result.systemout {{.value.op}}ContainSubstring 'step1 SUCCESS' + - result.systemout {{.value.op}}ContainSubstring 'step2 FAILURE' + # ranged steps + - result.systemout {{.value.op}}ContainSubstring 'exec (range=0) SUCCESS' + - result.systemout {{.value.op}}ContainSubstring 'exec (range=1) FAILURE' + - result.systemout {{.value.op}}ContainSubstring 'exec (range=2) SUCCESS' + # must assertions + - result.systemout {{.value.op}}ContainSubstring 'must1 FAILURE' + - result.systemout ShouldContainSubstring 'At least one required assertion failed, skipping remaining steps' + - result.systemout {{.value.op}}ContainSubstring '2 other steps were skipped' + - result.systemout ShouldNotContainSubstring 'must2' + - result.systemout ShouldNotContainSubstring 'must3' \ No newline at end of file diff --git a/types.go b/types.go index 3998fa43..28fa3f88 100644 --- a/types.go +++ b/types.go @@ -113,6 +113,7 @@ type TestCase struct { Time float64 `xml:"time,attr,omitempty" json:"time" yaml:"time,omitempty"` RawTestSteps []json.RawMessage `xml:"-" json:"steps" yaml:"steps"` testSteps []TestStep + TestStepResults []TestStepResult TestSuiteVars H `xml:"-" json:"-" yaml:"-"` Vars H `xml:"-" json:"-" yaml:"vars"` computedVars H @@ -123,6 +124,13 @@ type TestCase struct { IsEvaluated bool `xml:"-" json:"-" yaml:"-"` } +type TestStepResult struct { + Name string `xml:"name,attr" json:"name" yaml:"name"` + Skipped []Skipped `xml:"skipped,omitempty" json:"skipped" yaml:"skipped,omitempty"` + Errors []Failure `xml:"error,omitempty" json:"errors" yaml:"errors,omitempty"` + Failures []Failure `xml:"failure,omitempty" json:"failures" yaml:"failures,omitempty"` +} + // TestStep represents a testStep type TestStep map[string]interface{} @@ -195,22 +203,24 @@ type Failure struct { Message string `xml:"message,attr,omitempty" json:"message" yaml:"message,omitempty"` } -func newFailure(tc TestCase, stepNumber int, assertion string, err error) *Failure { +func newFailure(tc TestCase, stepNumber int, rangedIndex int, assertion string, err error) *Failure { var lineNumber = findLineNumber(tc.Classname, tc.originalName, stepNumber, assertion, -1) var value string if assertion != "" { - value = fmt.Sprintf(`Testcase %q, step #%d: Assertion %q failed. %s (%v:%d)`, + value = fmt.Sprintf(`Testcase %q, step #%d-%d: Assertion %q failed. %s (%v:%d)`, tc.originalName, stepNumber, + rangedIndex, RemoveNotPrintableChar(assertion), RemoveNotPrintableChar(err.Error()), tc.Classname, lineNumber, ) } else { - value = fmt.Sprintf(`Testcase %q, step #%d: %s (%v:%d)`, + value = fmt.Sprintf(`Testcase %q, step #%d-%d: %s (%v:%d)`, tc.originalName, stepNumber, + rangedIndex, RemoveNotPrintableChar(err.Error()), tc.Classname, lineNumber, diff --git a/types_executor.go b/types_executor.go index fa1e1030..45396a99 100644 --- a/types_executor.go +++ b/types_executor.go @@ -199,7 +199,7 @@ func (ux UserExecutor) ZeroValueResult() interface{} { return result } -func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn *TestCase, step TestStep) (interface{}, error) { +func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn *TestCase, tsIn *TestStepResult, step TestStep) (interface{}, error) { vrs := tcIn.TestSuiteVars.Clone() uxIn := runner.GetExecutor().(UserExecutor) @@ -239,7 +239,7 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn Debug(ctx, "running user executor %v", tc.Name) Debug(ctx, "with vars: %v", vrs) - v.runTestSteps(ctx, tc) + v.runTestSteps(ctx, tc, tsIn) computedVars, err := DumpString(tc.computedVars) if err != nil { diff --git a/venom.go b/venom.go index 02545305..f07e524f 100644 --- a/venom.go +++ b/venom.go @@ -74,7 +74,11 @@ func (v *Venom) Println(format string, a ...interface{}) { } func (v *Venom) PrintlnTrace(s string) { - v.Println("\t %s %s", trace("[trac]"), trace(s)) // nolint + v.PrintlnIndentedTrace(s, "") +} + +func (v *Venom) PrintlnIndentedTrace(s string, indent string) { + v.Println("\t %s%s %s", indent, trace("[trac]"), trace(s)) // nolint } func (v *Venom) AddVariables(variables map[string]interface{}) {