Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

espresso: Add Smart Retry for VDC #733

Merged
merged 6 commits into from
Mar 30, 2023
Merged

espresso: Add Smart Retry for VDC #733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
842ccd3
Add espresso retryier
FriggaHel Mar 30, 2023
a62e9e8
Add support for Both RDC/VDC
FriggaHel Mar 30, 2023
20cb170
Enhance teting scope
FriggaHel Mar 30, 2023
3a9e429
Fix small bug
FriggaHel Mar 30, 2023
5d9e667
Rename retrier
FriggaHel Mar 30, 2023
3670929
Simplify
FriggaHel Mar 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion internal/cmd/run/espresso.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,9 @@ func runEspressoInCloud(p espresso.Project, regio region.Region) (int, error) {
Framework: framework.Framework{Name: espresso.Kind},
Async: gFlags.async,
FailFast: gFlags.failFast,
Retrier: &retry.RDCRetrier{
Retrier: &retry.JunitRetrier{
RDCReader: &rdcClient,
VDCReader: &restoClient,
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion internal/cmd/run/xcuitest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func runXcuitestInCloud(p xcuitest.Project, regio region.Region) (int, error) {
Framework: framework.Framework{Name: xcuitest.Kind},
Async: gFlags.async,
FailFast: gFlags.failFast,
Retrier: &retry.RDCRetrier{
Retrier: &retry.JunitRetrier{
RDCReader: &rdcClient,
},
},
Expand Down
27 changes: 19 additions & 8 deletions internal/saucecloud/retry/rdcretrier.go → internal/saucecloud/retry/junitretrier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import (
"strings"
)

type RDCRetrier struct {
type JunitRetrier struct {
RDCReader job.Reader
VDCReader job.Reader
}

func getKeysFromMap(mp map[string]bool) []string {
Expand All @@ -38,30 +39,40 @@ func getFailedClasses(report junit.TestSuites) []string {
return getKeysFromMap(classes)
}

func (b *RDCRetrier) retryOnlyFailedClasses(jobOpts chan<- job.StartOptions, opt job.StartOptions, previous job.Job) {
content, err := b.RDCReader.GetJobAssetFileContent(context.Background(), previous.ID, junit.JunitFileName, previous.IsRDC)
func (b *JunitRetrier) retryOnlyFailedClasses(reader job.Reader, jobOpts chan<- job.StartOptions, opt job.StartOptions, previous job.Job) {
content, err := reader.GetJobAssetFileContent(context.Background(), previous.ID, junit.JunitFileName, previous.IsRDC)
if err != nil {
log.Debug().Err(err).Msgf(msg.UnableToFetchFile, junit.JunitFileName)
log.Info().Msg(msg.SkippingSmartRetries)
jobOpts <- opt
return
}
suites, err := junit.Parse(content)
if err != nil {
log.Debug().Err(err).Msg(msg.UnableToUnmarshallFile)
log.Debug().Err(err).Msgf(msg.UnableToUnmarshallFile, junit.JunitFileName)
log.Info().Msg(msg.SkippingSmartRetries)
jobOpts <- opt
return
}

classes := getFailedClasses(suites)
log.Info().Msgf(msg.RetryWithClasses, strings.Join(classes, ","))
log.Info().
Str("suite", opt.DisplayName).
Str("attempt", fmt.Sprintf("%d of %d", opt.Attempt+1, opt.Retries+1)).
Msgf(msg.RetryWithClasses, strings.Join(classes, ","))

opt.TestOptions["class"] = classes
jobOpts <- opt
}

func (b *RDCRetrier) Retry(jobOpts chan<- job.StartOptions, opt job.StartOptions, previous job.Job) {
if previous.IsRDC && opt.SmartRetry.FailedClassesOnly {
b.retryOnlyFailedClasses(jobOpts, opt, previous)
func (b *JunitRetrier) Retry(jobOpts chan<- job.StartOptions, opt job.StartOptions, previous job.Job) {
if b.RDCReader != nil && previous.IsRDC && opt.SmartRetry.FailedClassesOnly {
b.retryOnlyFailedClasses(b.RDCReader, jobOpts, opt, previous)
return
}

if b.VDCReader != nil && !previous.IsRDC && opt.SmartRetry.FailedClassesOnly {
b.retryOnlyFailedClasses(b.VDCReader, jobOpts, opt, previous)
return
}

Expand Down
287 changes: 287 additions & 0 deletions internal/saucecloud/retry/junitretrier_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,287 @@
package retry

import (
"context"
"errors"
"github.com/saucelabs/saucectl/internal/job"
"github.com/saucelabs/saucectl/internal/junit"
"github.com/saucelabs/saucectl/internal/mocks"
"github.com/stretchr/testify/assert"
"testing"
)

func TestAppsRetrier_Retry(t *testing.T) {
type args struct {
jobOpts chan job.StartOptions
opt job.StartOptions
previous job.Job
}
type init struct {
RDCReader job.Reader
VDCReader job.Reader
RetryRDC bool
RetryVDC bool
}
tests := []struct {
name string
init init
args args
expected job.StartOptions
}{
{
name: "Job is resent as-it if no RDC",
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"present"},
},
},
previous: job.Job{
IsRDC: false,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"present"},
},
},
},
{
name: "Job is untouched if there is no SmartRetries and is RDC",
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: false,
},
TestOptions: map[string]interface{}{
"class": []string{"present"},
},
},
previous: job.Job{
IsRDC: true,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"present"},
},
},
},
{
name: "Job retrying only failed suites if RDC + SmartRetry",
init: init{
RDCReader: &mocks.FakeJobReader{
ReadJobFn: nil,
PollJobFn: nil,
GetJobAssetFileNamesFn: nil,
GetJobAssetFileContentFn: func(ctx context.Context, jobID, fileName string) ([]byte, error) {
if jobID == "fake-job-id" && fileName == junit.JunitFileName {
return []byte("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n<testsuite>\n <testcase classname=\"Demo.Class1\">\n <failure>ERROR</failure>\n </testcase>\n <testcase classname=\"Demo.Class1\"/>\n <testcase classname=\"Demo.Class2\"/>\n <testcase classname=\"Demo.Class3\"/>\n</testsuite>\n"), nil
}
return []byte{}, errors.New("unknown file")
},
},
RetryRDC: true,
},
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
},
previous: job.Job{
ID: "fake-job-id",
IsRDC: true,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1"},
},
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
},
},
{
name: "Job not retrying if RDC and config is VDC + SmartRetry",
init: init{
RetryVDC: true,
},
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
},
previous: job.Job{
ID: "fake-job-id",
IsRDC: true,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
},
},
{
name: "Job is retrying when VDC + SmartRetry",
init: init{
VDCReader: &mocks.FakeJobReader{
ReadJobFn: nil,
PollJobFn: nil,
GetJobAssetFileNamesFn: nil,
GetJobAssetFileContentFn: func(ctx context.Context, jobID, fileName string) ([]byte, error) {
if jobID == "fake-job-id" && fileName == junit.JunitFileName {
return []byte("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n<testsuite>\n <testcase classname=\"Demo.Class1\">\n <failure>ERROR</failure>\n </testcase>\n <testcase classname=\"Demo.Class1\"/>\n <testcase classname=\"Demo.Class2\"/>\n <testcase classname=\"Demo.Class3\"/>\n</testsuite>\n"), nil
}
return []byte{}, errors.New("unknown file")
},
},
RetryVDC: true,
},
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
},
previous: job.Job{
ID: "fake-job-id",
IsRDC: false,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1"},
},
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
},
},
{
name: "Base Retry if junit is malformed",
init: init{
VDCReader: &mocks.FakeJobReader{
ReadJobFn: nil,
PollJobFn: nil,
GetJobAssetFileNamesFn: nil,
GetJobAssetFileContentFn: func(ctx context.Context, jobID, fileName string) ([]byte, error) {
if jobID == "fake-job-id" && fileName == junit.JunitFileName {
return []byte("malformed"), nil
}
return []byte{}, errors.New("unknown file")
},
},
RetryVDC: true,
},
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
},
previous: job.Job{
ID: "fake-job-id",
IsRDC: false,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
},
},
{
name: "Base Retry if getting junit.xml is failing",
init: init{
VDCReader: &mocks.FakeJobReader{
ReadJobFn: nil,
PollJobFn: nil,
GetJobAssetFileNamesFn: nil,
GetJobAssetFileContentFn: func(ctx context.Context, jobID, fileName string) ([]byte, error) {
if jobID == "fake-job-id" && fileName == junit.JunitFileName {
return []byte("malformed"), nil
}
return []byte{}, errors.New("unknown file")
},
},
RetryVDC: true,
},
args: args{
jobOpts: make(chan job.StartOptions),
opt: job.StartOptions{
DisplayName: "Dummy Test",
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
},
previous: job.Job{
ID: "fake-buggy-job-id",
IsRDC: false,
},
},
expected: job.StartOptions{
DisplayName: "Dummy Test",
TestOptions: map[string]interface{}{
"class": []string{"Demo.Class1", "Demo.Class2", "Demo.Class3"},
},
SmartRetry: job.SmartRetry{
FailedClassesOnly: true,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &JunitRetrier{
RDCReader: tt.init.RDCReader,
VDCReader: tt.init.VDCReader,
}
go b.Retry(tt.args.jobOpts, tt.args.opt, tt.args.previous)
newOpt := <-tt.args.jobOpts
assert.Equal(t, tt.expected, newOpt)
})
}
}
Loading