From 5b64dc636eaaf71c3cb2666dff9c330be67a0375 Mon Sep 17 00:00:00 2001 From: Tian Feng Date: Fri, 3 Nov 2023 15:21:06 -0700 Subject: [PATCH] feat: Add JSON reporter for Sauce Orchestrate (#849) * feat: Add JSON reporter for Sauce Orchestrate * add comments to DownloadArtifact method * make sure artifacts in report always exist * update JSON schema for image runner * refine appending artifacts logic * simplify * simplify again * refine comments * Update internal/saucecloud/imagerunner.go Co-authored-by: Alex Plischke * revise JSON schema * refine JSON reporter and reorg TestResult * Update internal/saucecloud/imagerunner.go Co-authored-by: Alex Plischke * Update internal/report/json/json.go Co-authored-by: Alex Plischke * Update api/v1alpha/framework/imagerunner.schema.json Co-authored-by: Alex Plischke * sync schema --------- Co-authored-by: Alex Plischke --- api/saucectl.schema.json | 31 ++++++++++++++++++- api/v1alpha/framework/imagerunner.schema.json | 29 +++++++++++++++++ api/v1alpha/subschema/reporters.schema.json | 2 +- internal/cmd/run/imagerunner.go | 22 ++++++++++--- internal/http/resto.go | 3 +- internal/imagerunner/config.go | 1 + internal/report/json/json.go | 26 +++------------- internal/report/report.go | 5 +-- internal/saucecloud/imagerunner.go | 31 +++++++++++++------ 9 files changed, 111 insertions(+), 39 deletions(-) diff --git a/api/saucectl.schema.json b/api/saucectl.schema.json index 7ea6e9722..a6b5883da 100644 --- a/api/saucectl.schema.json +++ b/api/saucectl.schema.json @@ -289,7 +289,7 @@ "filename": { "description": "Filename for the generated JSON report.", "type": "string", - "default": "saucectl-test-result.xml" + "default": "saucectl-report.json" } } }, @@ -2845,6 +2845,32 @@ "required": [ "name" ] + }, + "reporters": { + "description": "Supported saucectl reporters.", + "type": "object", + "properties": { + "json": { + "type": "object", + "description": "The JSON reporter creates single report of all executed saucectl suites.", + "properties": { + "enabled": { + "description": "Toggles the reporter on/off.", + "type": "boolean" + }, + "webhookURL": { + "description": "Webhook URL to pass JSON report.", + "type": "string" + }, + "filename": { + "description": "Filename for the generated JSON report.", + "type": "string", + "default": "saucectl-report.json" + } + } + }, + "additionalProperties": false + } } }, "properties": { @@ -2868,6 +2894,9 @@ "items": { "$ref": "#/allOf/9/then/definitions/suite" } + }, + "reporters": { + "$ref": "#/allOf/9/then/definitions/reporters" } }, "required": [ diff --git a/api/v1alpha/framework/imagerunner.schema.json b/api/v1alpha/framework/imagerunner.schema.json index b56af2466..5767cd5ed 100644 --- a/api/v1alpha/framework/imagerunner.schema.json +++ b/api/v1alpha/framework/imagerunner.schema.json @@ -197,6 +197,32 @@ "required": [ "name" ] + }, + "reporters": { + "description": "Supported saucectl reporters.", + "type": "object", + "properties": { + "json": { + "type": "object", + "description": "The JSON reporter creates single report of all executed saucectl suites.", + "properties": { + "enabled": { + "description": "Toggles the reporter on/off.", + "type": "boolean" + }, + "webhookURL": { + "description": "Webhook URL to pass JSON report.", + "type": "string" + }, + "filename": { + "description": "Filename for the generated JSON report.", + "type": "string", + "default": "saucectl-report.json" + } + } + }, + "additionalProperties": false + } } }, "properties": { @@ -220,6 +246,9 @@ "items": { "$ref": "#/definitions/suite" } + }, + "reporters": { + "$ref": "#/definitions/reporters" } }, "required": [ diff --git a/api/v1alpha/subschema/reporters.schema.json b/api/v1alpha/subschema/reporters.schema.json index f7252f6ab..3035d88c3 100644 --- a/api/v1alpha/subschema/reporters.schema.json +++ b/api/v1alpha/subschema/reporters.schema.json @@ -37,7 +37,7 @@ "filename": { "description": "Filename for the generated JSON report.", "type": "string", - "default": "saucectl-test-result.xml" + "default": "saucectl-report.json" } } }, diff --git a/internal/cmd/run/imagerunner.go b/internal/cmd/run/imagerunner.go index aada5a850..abc8d29bd 100644 --- a/internal/cmd/run/imagerunner.go +++ b/internal/cmd/run/imagerunner.go @@ -8,6 +8,7 @@ import ( "github.com/saucelabs/saucectl/internal/imagerunner" "github.com/saucelabs/saucectl/internal/region" "github.com/saucelabs/saucectl/internal/report" + "github.com/saucelabs/saucectl/internal/report/json" "github.com/saucelabs/saucectl/internal/report/table" "github.com/saucelabs/saucectl/internal/saucecloud" "github.com/saucelabs/saucectl/internal/segment" @@ -50,14 +51,27 @@ func runImageRunner(cmd *cobra.Command) (int, error) { _ = tracker.Close() }() + reporters := []report.Reporter{ + &table.Reporter{ + Dst: os.Stdout, + }, + } + if !gFlags.async { + if p.Reporters.JSON.Enabled { + reporters = append(reporters, &json.Reporter{ + WebhookURL: p.Reporters.JSON.WebhookURL, + Filename: p.Reporters.JSON.Filename, + }) + } + } + + cleanupArtifacts(p.Artifacts) r := saucecloud.ImgRunner{ Project: p, RunnerService: &imageRunnerClient, TunnelService: &restoClient, - Reporters: []report.Reporter{&table.Reporter{ - Dst: os.Stdout, - }}, - Async: gFlags.async, + Reporters: reporters, + Async: gFlags.async, } return r.RunProject() } diff --git a/internal/http/resto.go b/internal/http/resto.go index 1d95b4d86..4f1b78987 100644 --- a/internal/http/resto.go +++ b/internal/http/resto.go @@ -361,9 +361,10 @@ func (c *Resto) DownloadArtifact(jobID, suiteName string, realDevice bool) []str for _, f := range files { for _, pattern := range c.ArtifactConfig.Match { if glob.Glob(pattern, f) { - artifacts = append(artifacts, filepath.Join(targetDir, f)) if err := c.downloadArtifact(targetDir, jobID, f); err != nil { log.Error().Err(err).Msgf("Failed to download file: %s", f) + } else { + artifacts = append(artifacts, filepath.Join(targetDir, f)) } break } diff --git a/internal/imagerunner/config.go b/internal/imagerunner/config.go index 854ccc5cd..d2852848e 100644 --- a/internal/imagerunner/config.go +++ b/internal/imagerunner/config.go @@ -34,6 +34,7 @@ type Project struct { DryRun bool `yaml:"-" json:"-"` Env map[string]string `yaml:"env,omitempty" json:"env"` EnvFlag map[string]string `yaml:"-" json:"-"` + Reporters config.Reporters `yaml:"reporters,omitempty" json:"-"` } type Defaults struct { diff --git a/internal/report/json/json.go b/internal/report/json/json.go index 76d20cad3..71782f09c 100644 --- a/internal/report/json/json.go +++ b/internal/report/json/json.go @@ -25,24 +25,23 @@ func (r *Reporter) Add(t report.TestResult) { // Render sends the result to specified webhook WebhookURL and log the result to the specified json file func (r *Reporter) Render() { - r.cleanup() body, err := json.Marshal(r.Results) if err != nil { - log.Error().Msgf("failed to generate test result (%v)", err) + log.Err(err).Msg("failed to generate test result.") return } if r.WebhookURL != "" { resp, err := http.Post(r.WebhookURL, "application/json", bytes.NewBuffer(body)) if err != nil { - log.Error().Err(err).Str("webhook", r.WebhookURL).Msg("failed to send test result to webhook.") + log.Err(err).Str("webhook", r.WebhookURL).Msg("failed to send test result to webhook.") } else { webhookBody, _ := io.ReadAll(resp.Body) if resp.StatusCode >= http.StatusBadRequest { - log.Error().Str("webhook", r.WebhookURL).Msgf("failed to send test result to webhook, status: '%d', msg:'%v'", resp.StatusCode, string(webhookBody)) + log.Error().Str("webhook", r.WebhookURL).Msgf("failed to send test result to webhook, status: %d, msg: %q.", resp.StatusCode, string(webhookBody)) } if resp.StatusCode%100 == 2 { - log.Info().Str("webhook", r.WebhookURL).Msgf("test result has been sent successfully to webhook, msg: '%v'.", string(webhookBody)) + log.Info().Str("webhook", r.WebhookURL).Msgf("test result has been sent successfully to webhook, msg: %q.", string(webhookBody)) } } } @@ -50,26 +49,11 @@ func (r *Reporter) Render() { if r.Filename != "" { err = os.WriteFile(r.Filename, body, 0666) if err != nil { - log.Error().Err(err).Msgf("failed to write test result to %s", r.Filename) + log.Err(err).Msgf("failed to write test result to %q.", r.Filename) } } } -// cleanup removes any information that isn't relevant in the rendered report. Particularly when it comes to -// artifacts, this reporter is only interested in those that have been persisted to the file system. -func (r *Reporter) cleanup() { - for i, result := range r.Results { - var artifacts []report.Artifact - for _, a := range result.Artifacts { - if a.FilePath == "" { - continue - } - artifacts = append(artifacts, a) - } - r.Results[i].Artifacts = artifacts - } -} - // Reset resets the reporter to its initial state. This action will delete all test results. func (r *Reporter) Reset() { r.Results = make([]report.TestResult, 0) diff --git a/internal/report/report.go b/internal/report/report.go index a2b1ab42a..83a371e5a 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -32,10 +32,11 @@ type TestResult struct { Browser string `json:"browser,omitempty"` Platform string `json:"platform"` DeviceName string `json:"deviceName,omitempty"` - URL string `json:"url"` + URL string `json:"url,omitempty"` Artifacts []Artifact `json:"artifacts,omitempty"` - Origin string `json:"origin"` + Origin string `json:"origin,omitempty"` BuildURL string `json:"buildURL,omitempty"` + RunID string `json:"runID,omitempty"` RDC bool `json:"-"` TimedOut bool `json:"-"` PassThreshold bool `json:"-"` diff --git a/internal/saucecloud/imagerunner.go b/internal/saucecloud/imagerunner.go index 370f9bcd0..7a364f4b8 100644 --- a/internal/saucecloud/imagerunner.go +++ b/internal/saucecloud/imagerunner.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/signal" + "path/filepath" "reflect" "time" @@ -318,7 +319,11 @@ func (r *ImgRunner) collectResults(results chan execResult, expected int) bool { r.PrintResult(res) r.PrintLogs(res.runID, res.name) - r.DownloadArtifacts(res.runID, res.name, res.status, res.err != nil) + files := r.DownloadArtifacts(res.runID, res.name, res.status, res.err != nil) + var artifacts []report.Artifact + for _, f := range files { + artifacts = append(artifacts, report.Artifact{FilePath: f}) + } for _, r := range r.Reporters { r.Add(report.TestResult{ @@ -327,6 +332,9 @@ func (r *ImgRunner) collectResults(results chan execResult, expected int) bool { StartTime: res.startTime, EndTime: res.endTime, Status: res.status, + Artifacts: artifacts, + Platform: "Linux", + RunID: res.runID, Attempts: []report.Attempt{{ ID: res.runID, Duration: res.duration, @@ -391,51 +399,56 @@ func (r *ImgRunner) PollRun(ctx context.Context, id string, lastStatus string) ( } } -func (r *ImgRunner) DownloadArtifacts(runnerID, suiteName, status string, passed bool) { +// DownloadArtifact downloads a zipped archive of artifacts and extracts the required files. +func (r *ImgRunner) DownloadArtifacts(runnerID, suiteName, status string, passed bool) []string { if r.Async || runnerID == "" || status == imagerunner.StateCancelled || !r.Project.Artifacts.Download.When.IsNow(passed) { - return + return nil } dir, err := config.GetSuiteArtifactFolder(suiteName, r.Project.Artifacts.Download) if err != nil { log.Err(err).Msg("Unable to create artifacts folder.") - return + return nil } log.Info().Msg("Downloading artifacts archive") reader, err := r.RunnerService.DownloadArtifacts(r.ctx, runnerID) if err != nil { log.Err(err).Str("suite", suiteName).Msg("Failed to fetch artifacts.") - return + return nil } defer reader.Close() fileName, err := fileio.CreateTemp(reader) if err != nil { log.Err(err).Str("suite", suiteName).Msg("Failed to download artifacts content.") - return + return nil } defer os.Remove(fileName) zf, err := zip.OpenReader(fileName) if err != nil { - log.Error().Msgf("Unable to open zip file %s: %s", fileName, err) - return + log.Err(err).Msgf("Unable to open zip file %q", fileName) + return nil } defer zf.Close() + var artifacts []string for _, f := range zf.File { for _, pattern := range r.Project.Artifacts.Download.Match { if glob.Glob(pattern, f.Name) { if err = szip.Extract(dir, f); err != nil { - log.Error().Msgf("Unable to extract file '%s': %s", f.Name, err) + log.Err(err).Msgf("Unable to extract file %q", f.Name) + } else { + artifacts = append(artifacts, filepath.Join(dir, f.Name)) } break } } } + return artifacts } func (r *ImgRunner) PrintResult(res execResult) {