Skip to content

Commit

Permalink
feat: detailed steps output (#487)
Browse files Browse the repository at this point in the history 
* feat: detailed steps output

Signed-off-by: GitHub <noreply@github.com>
  • Loading branch information
@lowlighter
lowlighter authored Aug 1, 2022
1 parent 00ead58 commit d2680fe
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 60 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,8 @@ Flags and their equivalent with environment variables usage:
- `-v` flag is equivalent to `VENOM_VERBOSE=1` environment variable
- `-vv` flag is equivalent to `VENOM_VERBOSE=2` environment variable

It is possible to set `NO_COLOR=1` environment variable to disable colors from output.

## Use a configuration file

You can define the Venom settings using a configuration file `.venomrc`. This configuration file should be placed in the current directory or in the `home` directory.
Expand Down
30 changes: 15 additions & 15 deletions assertion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ type assertionsApplied struct {
systemerr string
}

func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep, defaultAssertions *StepAssertions) assertionsApplied {
func applyAssertions(r interface{}, tc TestCase, stepNumber int, rangedIndex int, step TestStep, defaultAssertions *StepAssertions) assertionsApplied {
var sa StepAssertions
var errors []Failure
var failures []Failure
Expand All @@ -48,7 +48,7 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep,

isOK := true
for _, assertion := range sa.Assertions {
errs, fails := check(tc, stepNumber, assertion, executorResult)
errs, fails := check(tc, stepNumber, rangedIndex, assertion, executorResult)
if errs != nil {
errors = append(errors, *errs)
isOK = false
Expand Down Expand Up @@ -116,26 +116,26 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti
}

// check selects the correct assertion function to call depending on typing provided by user
func check(tc TestCase, stepNumber int, assertion Assertion, r interface{}) (*Failure, *Failure) {
func check(tc TestCase, stepNumber int, rangedIndex int, assertion Assertion, r interface{}) (*Failure, *Failure) {
var errs *Failure
var fails *Failure
switch t := assertion.(type) {
case string:
errs, fails = checkString(tc, stepNumber, assertion.(string), r)
errs, fails = checkString(tc, stepNumber, rangedIndex, assertion.(string), r)
case map[string]interface{}:
errs, fails = checkBranch(tc, stepNumber, assertion.(map[string]interface{}), r)
errs, fails = checkBranch(tc, stepNumber, rangedIndex, assertion.(map[string]interface{}), r)
default:
errs = newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion format: %v", t))
errs = newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("unsupported assertion format: %v", t))
}
return errs, fails
}

// checkString evaluate a complex assertion containing logical operators
// it recursively calls checkAssertion for each operand
func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r interface{}) (*Failure, *Failure) {
func checkBranch(tc TestCase, stepNumber int, rangedIndex int, branch map[string]interface{}, r interface{}) (*Failure, *Failure) {
// Extract logical operator
if len(branch) != 1 {
return newFailure(tc, stepNumber, "", fmt.Errorf("expected exactly 1 logical operator but %d were provided", len(branch))), nil
return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("expected exactly 1 logical operator but %d were provided", len(branch))), nil
}
var operator string
for k := range branch {
Expand All @@ -148,7 +148,7 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i
case []interface{}:
operands = branch[operator].([]interface{})
default:
return newFailure(tc, stepNumber, "", fmt.Errorf("expected %s operands to be an []interface{}, got %v", operator, t)), nil
return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("expected %s operands to be an []interface{}, got %v", operator, t)), nil
}
if len(operands) == 0 {
return nil, nil
Expand All @@ -161,7 +161,7 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i
assertionsCount := len(operands)
assertionsSuccess := 0
for _, assertion := range operands {
errs, fails := check(tc, stepNumber, assertion, r)
errs, fails := check(tc, stepNumber, rangedIndex, assertion, r)
if errs != nil {
errsBuf = append(errsBuf, *errs)
}
Expand Down Expand Up @@ -198,23 +198,23 @@ func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r i
err = fmt.Errorf("some assertions succeeded but expected none to suceed:\n%s\n", strings.Join(results, "\n"))
}
default:
return newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion operator %s", operator)), nil
return newFailure(tc, stepNumber, rangedIndex, "", fmt.Errorf("unsupported assertion operator %s", operator)), nil
}
if err != nil {
return nil, newFailure(tc, stepNumber, "", err)
return nil, newFailure(tc, stepNumber, rangedIndex, "", err)
}
return nil, nil
}

// checkString evaluate a single string assertion
func checkString(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) {
func checkString(tc TestCase, stepNumber int, rangedIndex int, assertion string, r interface{}) (*Failure, *Failure) {
assert, err := parseAssertions(context.Background(), assertion, r)
if err != nil {
return nil, newFailure(tc, stepNumber, assertion, err)
return nil, newFailure(tc, stepNumber, rangedIndex, assertion, err)
}

if err := assert.Func(assert.Actual, assert.Args...); err != nil {
failure := newFailure(tc, stepNumber, assertion, err)
failure := newFailure(tc, stepNumber, rangedIndex, assertion, err)
failure.AssertionRequired = assert.Required
return nil, failure
}
Expand Down
92 changes: 80 additions & 12 deletions process_testcase.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strconv"
"strings"

"github.com/fatih/color"
"github.com/ghodss/yaml"
"github.com/ovh/cds/sdk/interpolate"
"github.com/pkg/errors"
Expand Down Expand Up @@ -146,10 +147,11 @@ func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) {
}

defer Info(ctx, "Ending testcase")
v.runTestSteps(ctx, tc)
v.runTestSteps(ctx, tc, nil)
}

func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepResult) {

results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v")
if err != nil {
Error(ctx, "unable to evaluate \"skip\" assertions: %v", err)
Expand All @@ -166,6 +168,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {

var knowExecutors = map[string]struct{}{}
var previousStepVars = H{}
fromUserExecutor := tsIn != nil

for stepNumber, rawStep := range tc.RawTestSteps {
stepVars := tc.Vars.Clone()
Expand All @@ -181,8 +184,10 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
}

for rangedIndex, rangedData := range ranged.Items {
tc.TestStepResults = append(tc.TestStepResults, TestStepResult{})
ts := &tc.TestStepResults[len(tc.TestStepResults)-1]
if ranged.Enabled {
Debug(ctx, "processing step %d", rangedIndex)
Debug(ctx, "processing range index: %d", rangedIndex)
stepVars.Add("index", rangedIndex)
stepVars.Add("key", rangedData.Key)
stepVars.Add("value", rangedData.Value)
Expand Down Expand Up @@ -225,7 +230,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
}
}

Info(ctx, "Step #%d content is: %q", stepNumber, content)
Info(ctx, "Step #%d-%d content is: %q", stepNumber, rangedIndex, content)
var step TestStep
if err := yaml.Unmarshal([]byte(content), &step); err != nil {
tc.AppendError(err)
Expand Down Expand Up @@ -261,38 +266,44 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
}
}

result := v.RunTestStep(ctx, e, tc, stepNumber, step)
printStepName := v.Verbose >= 2 && !fromUserExecutor
v.setTestStepName(ts, e, step, &ranged, &rangedData, printStepName)

result := v.RunTestStep(ctx, e, tc, ts, stepNumber, rangedIndex, step)
mapResult := GetExecutorResult(result)
previousStepVars.AddAll(H(mapResult))

tc.testSteps = append(tc.testSteps, step)

var hasFailed bool
var isRequired bool
if len(tc.Failures) > 0 {
for _, f := range tc.Failures {
if len(ts.Failures) > 0 {
for _, f := range ts.Failures {
Warning(ctx, "%v", f)
isRequired = isRequired || f.AssertionRequired
}
hasFailed = true
}

if len(tc.Errors) > 0 {
if len(ts.Errors) > 0 {
Error(ctx, "Errors: ")
for _, e := range tc.Errors {
for _, e := range ts.Errors {
Error(ctx, "%v", e)
}
hasFailed = true
}

if hasFailed {
if isRequired {
failure := newFailure(*tc, stepNumber, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps"))
tc.Failures = append(tc.Failures, *failure)
failure := newFailure(*tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps"))
ts.appendFailure(tc, *failure)
v.printTestStepResult(tc, ts, tsIn, stepNumber, true)
return
}
break
v.printTestStepResult(tc, ts, tsIn, stepNumber, false)
continue
}
v.printTestStepResult(tc, ts, tsIn, stepNumber, false)

allVars := tc.Vars.Clone()
allVars.AddAll(tc.computedVars.Clone())
Expand All @@ -310,6 +321,63 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
}
}

//Set test step name (defaults to executor name, excepted if it got a "name" attribute. in range, also print key)
func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestStep, ranged *Range, rangedData *RangeData, print bool) {
name := e.Name()
if value, ok := step["name"]; ok {
switch value := value.(type) {
case string:
name = value
}
}
if ranged.Enabled {
name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key)
}
ts.Name = name

if print {
v.Print(" \t\t• %s", ts.Name)
}
}

//Print a single step result (if verbosity is enabled)
func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) {
if v.Verbose >= 2 {
fromUserExecutor := tsIn != nil
var red = color.New(color.FgRed).SprintFunc()
var green = color.New(color.FgGreen).SprintFunc()
var gray = color.New(color.Attribute(90)).SprintFunc()

//Within an user executor, we instead transfer errors to original test step which will print it for use
if fromUserExecutor {
tsIn.appendError(tc, ts.Errors...)
tsIn.appendFailure(tc, ts.Failures...)
} else { //Else print step status
if len(ts.Skipped) > 0 {
v.Println(" %s", gray("SKIPPED"))
} else if len(ts.Failures) > 0 || len(ts.Errors) > 0 {
v.Println(" %s", red("FAILURE"))
for _, f := range ts.Failures {
v.Println(" \t\t %s", red(f))
}
for _, f := range ts.Errors {
v.Println(" \t\t %s", red(f.Value))
}
if mustAssertionFailed {
skipped := len(tc.RawTestSteps) - stepNumber - 1
if skipped == 1 {
v.Println(" \t\t %s", gray(fmt.Sprintf("%d other step was skipped", skipped)))
} else {
v.Println(" \t\t %s", gray(fmt.Sprintf("%d other steps were skipped", skipped)))
}
}
} else {
v.Println(" %s", green("SUCCESS"))
}
}
}
}

//Parse and format range data to allow iterations over user data
func parseRanged(ctx context.Context, rawStep []byte, stepVars H) (Range, error) {

Expand Down
42 changes: 27 additions & 15 deletions process_teststep.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type dumpFile struct {
}

//RunTestStep executes a venom testcase is a venom context
func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, stepNumber int, step TestStep) interface{} {
func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} {
ctx = context.WithValue(ctx, ContextKey("executor"), e.Name())

var assertRes assertionsApplied
Expand All @@ -33,12 +33,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
}

var err error
result, err = v.runTestStepExecutor(ctx, e, tc, step)
result, err = v.runTestStepExecutor(ctx, e, tc, ts, step)
if err != nil {
// we save the failure only if it's the last attempt
if retry == e.Retry() {
failure := newFailure(*tc, stepNumber, "", err)
tc.Failures = append(tc.Failures, *failure)
failure := newFailure(*tc, stepNumber, rangedIndex, "", err)
ts.appendFailure(tc, *failure)
}
continue
}
Expand All @@ -62,7 +62,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
if oDir == "" {
oDir = "."
}
filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber))
filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex))

if err := os.WriteFile(filename, []byte(output), 0644); err != nil {
return fmt.Errorf("Error while creating file %s: %v", filename, err)
Expand Down Expand Up @@ -97,12 +97,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,

if result == nil {
Debug(ctx, "empty testcase, applying assertions on variables: %v", AllVarsFromCtx(ctx))
assertRes = applyAssertions(AllVarsFromCtx(ctx), *tc, stepNumber, step, nil)
assertRes = applyAssertions(AllVarsFromCtx(ctx), *tc, stepNumber, rangedIndex, step, nil)
} else {
if h, ok := e.(executorWithDefaultAssertions); ok {
assertRes = applyAssertions(result, *tc, stepNumber, step, h.GetDefaultAssertions())
assertRes = applyAssertions(result, *tc, stepNumber, rangedIndex, step, h.GetDefaultAssertions())
} else {
assertRes = applyAssertions(result, *tc, stepNumber, step, nil)
assertRes = applyAssertions(result, *tc, stepNumber, rangedIndex, step, nil)
}
}

Expand All @@ -116,29 +116,29 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
return fmt.Errorf("Error while evaluating retry condition: %v", err)
}
if len(failures) > 0 {
failure := newFailure(*tc, stepNumber, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-retry))
failure := newFailure(*tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-retry))
tc.Failures = append(tc.Failures, *failure)
break
}
}

tc.Errors = append(tc.Errors, assertRes.errors...)
tc.Failures = append(tc.Failures, assertRes.failures...)
ts.appendError(tc, assertRes.errors...)
ts.appendFailure(tc, assertRes.failures...)
if retry > 1 && (len(assertRes.failures) > 0 || len(assertRes.errors) > 0) {
tc.Failures = append(tc.Failures, Failure{Value: fmt.Sprintf("It's a failure after %d attempts", retry)})
ts.appendFailure(tc, Failure{Value: fmt.Sprintf("It's a failure after %d attempts", retry)})
}
tc.Systemout.Value += assertRes.systemout
tc.Systemerr.Value += assertRes.systemerr

return result
}

func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, step TestStep) (interface{}, error) {
func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) {
ctx = context.WithValue(ctx, ContextKey("executor"), e.Name())

if e.Timeout() == 0 {
if e.Type() == "user" {
return v.RunUserExecutor(ctx, e, tc, step)
return v.RunUserExecutor(ctx, e, tc, ts, step)
}
return e.Run(ctx, step)
}
Expand All @@ -152,7 +152,7 @@ func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *T
var err error
var result interface{}
if e.Type() == "user" {
result, err = v.RunUserExecutor(ctx, e, tc, step)
result, err = v.RunUserExecutor(ctx, e, tc, ts, step)
} else {
result, err = e.Run(ctx, step)
}
Expand All @@ -172,3 +172,15 @@ func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *T
return nil, fmt.Errorf("Timeout after %d second(s)", e.Timeout())
}
}

//Append an error to a test step and its associated test case
func (ts *TestStepResult) appendError(tc *TestCase, failure ...Failure) {
ts.Errors = append(ts.Errors, failure...)
tc.Errors = append(tc.Errors, failure...)
}

//Append an assertion failure to a test step and its associated test case
func (ts *TestStepResult) appendFailure(tc *TestCase, failure ...Failure) {
ts.Failures = append(ts.Failures, failure...)
tc.Failures = append(tc.Failures, failure...)
}
Loading

0 comments on commit d2680fe

Please sign in to comment.