feat: Add JSON reporter for Sauce Orchestrate (#849)

* feat: Add JSON reporter for Sauce Orchestrate * add comments to DownloadArtifact method * make sure artifacts in report always exist * update JSON schema for image runner * refine appending artifacts logic * simplify * simplify again * refine comments * Update internal/saucecloud/imagerunner.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * revise JSON schema * refine JSON reporter and reorg TestResult * Update internal/saucecloud/imagerunner.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * Update internal/report/json/json.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * Update api/v1alpha/framework/imagerunner.schema.json Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * sync schema --------- Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com>
saucelabs · Nov 3, 2023 · 5b64dc6 · 5b64dc6
1 parent 5a44562
commit 5b64dc6
Show file tree

Hide file tree

Showing 9 changed files with 111 additions and 39 deletions.
diff --git a/api/saucectl.schema.json b/api/saucectl.schema.json
@@ -289,7 +289,7 @@
                       "filename": {
                         "description": "Filename for the generated JSON report.",
                         "type": "string",
-                        "default": "saucectl-test-result.xml"
+                        "default": "saucectl-report.json"
                       }
                     }
                   },
@@ -2845,6 +2845,32 @@
             "required": [
               "name"
             ]
+          },
+          "reporters": {
+            "description": "Supported saucectl reporters.",
+            "type": "object",
+            "properties": {
+              "json": {
+                "type": "object",
+                "description": "The JSON reporter creates single report of all executed saucectl suites.",
+                "properties": {
+                  "enabled": {
+                    "description": "Toggles the reporter on/off.",
+                    "type": "boolean"
+                  },
+                  "webhookURL": {
+                    "description": "Webhook URL to pass JSON report.",
+                    "type": "string"
+                  },
+                  "filename": {
+                    "description": "Filename for the generated JSON report.",
+                    "type": "string",
+                    "default": "saucectl-report.json"
+                  }
+                }
+              },
+              "additionalProperties": false
+            }
           }
         },
         "properties": {
@@ -2868,6 +2894,9 @@
             "items": {
               "$ref": "#/allOf/9/then/definitions/suite"
             }
+          },
+          "reporters": {
+            "$ref": "#/allOf/9/then/definitions/reporters"
           }
         },
         "required": [

diff --git a/api/v1alpha/framework/imagerunner.schema.json b/api/v1alpha/framework/imagerunner.schema.json
@@ -197,6 +197,32 @@
       "required": [
         "name"
       ]
+    },
+    "reporters": {
+      "description": "Supported saucectl reporters.",
+      "type": "object",
+      "properties": {
+        "json": {
+          "type": "object",
+          "description": "The JSON reporter creates single report of all executed saucectl suites.",
+          "properties": {
+            "enabled": {
+              "description": "Toggles the reporter on/off.",
+              "type": "boolean"
+            },
+            "webhookURL": {
+              "description": "Webhook URL to pass JSON report.",
+              "type": "string"
+            },
+            "filename": {
+              "description": "Filename for the generated JSON report.",
+              "type": "string",
+              "default": "saucectl-report.json"
+            }
+          }
+        },
+        "additionalProperties": false
+      }
     }
   },
   "properties": {
@@ -220,6 +246,9 @@
       "items": {
         "$ref": "#/definitions/suite"
       }
+    },
+    "reporters": {
+      "$ref":  "#/definitions/reporters"
     }
   },
   "required": [

diff --git a/api/v1alpha/subschema/reporters.schema.json b/api/v1alpha/subschema/reporters.schema.json
@@ -37,7 +37,7 @@
             "filename": {
               "description": "Filename for the generated JSON report.",
               "type": "string",
-              "default": "saucectl-test-result.xml"
+              "default": "saucectl-report.json"
             }
           }
         },

diff --git a/internal/cmd/run/imagerunner.go b/internal/cmd/run/imagerunner.go
@@ -8,6 +8,7 @@ import (
 	"github.com/saucelabs/saucectl/internal/imagerunner"
 	"github.com/saucelabs/saucectl/internal/region"
 	"github.com/saucelabs/saucectl/internal/report"
+	"github.com/saucelabs/saucectl/internal/report/json"
 	"github.com/saucelabs/saucectl/internal/report/table"
 	"github.com/saucelabs/saucectl/internal/saucecloud"
 	"github.com/saucelabs/saucectl/internal/segment"
@@ -50,14 +51,27 @@ func runImageRunner(cmd *cobra.Command) (int, error) {
 		_ = tracker.Close()
 	}()
 
+	reporters := []report.Reporter{
+		&table.Reporter{
+			Dst: os.Stdout,
+		},
+	}
+	if !gFlags.async {
+		if p.Reporters.JSON.Enabled {
+			reporters = append(reporters, &json.Reporter{
+				WebhookURL: p.Reporters.JSON.WebhookURL,
+				Filename:   p.Reporters.JSON.Filename,
+			})
+		}
+	}
+
+	cleanupArtifacts(p.Artifacts)
 	r := saucecloud.ImgRunner{
 		Project:       p,
 		RunnerService: &imageRunnerClient,
 		TunnelService: &restoClient,
-		Reporters: []report.Reporter{&table.Reporter{
-			Dst: os.Stdout,
-		}},
-		Async: gFlags.async,
+		Reporters:     reporters,
+		Async:         gFlags.async,
 	}
 	return r.RunProject()
 }

diff --git a/internal/http/resto.go b/internal/http/resto.go
@@ -361,9 +361,10 @@ func (c *Resto) DownloadArtifact(jobID, suiteName string, realDevice bool) []str
 	for _, f := range files {
 		for _, pattern := range c.ArtifactConfig.Match {
 			if glob.Glob(pattern, f) {
-				artifacts = append(artifacts, filepath.Join(targetDir, f))
 				if err := c.downloadArtifact(targetDir, jobID, f); err != nil {
 					log.Error().Err(err).Msgf("Failed to download file: %s", f)
+				} else {
+					artifacts = append(artifacts, filepath.Join(targetDir, f))
 				}
 				break
 			}

diff --git a/internal/imagerunner/config.go b/internal/imagerunner/config.go
@@ -34,6 +34,7 @@ type Project struct {
 	DryRun         bool               `yaml:"-" json:"-"`
 	Env            map[string]string  `yaml:"env,omitempty" json:"env"`
 	EnvFlag        map[string]string  `yaml:"-" json:"-"`
+	Reporters      config.Reporters   `yaml:"reporters,omitempty" json:"-"`
 }
 
 type Defaults struct {

diff --git a/internal/report/json/json.go b/internal/report/json/json.go
@@ -25,51 +25,35 @@ func (r *Reporter) Add(t report.TestResult) {
 
 // Render sends the result to specified webhook WebhookURL and log the result to the specified json file
 func (r *Reporter) Render() {
-	r.cleanup()
 	body, err := json.Marshal(r.Results)
 	if err != nil {
-		log.Error().Msgf("failed to generate test result (%v)", err)
+		log.Err(err).Msg("failed to generate test result.")
 		return
 	}
 
 	if r.WebhookURL != "" {
 		resp, err := http.Post(r.WebhookURL, "application/json", bytes.NewBuffer(body))
 		if err != nil {
-			log.Error().Err(err).Str("webhook", r.WebhookURL).Msg("failed to send test result to webhook.")
+			log.Err(err).Str("webhook", r.WebhookURL).Msg("failed to send test result to webhook.")
 		} else {
 			webhookBody, _ := io.ReadAll(resp.Body)
 			if resp.StatusCode >= http.StatusBadRequest {
-				log.Error().Str("webhook", r.WebhookURL).Msgf("failed to send test result to webhook, status: '%d', msg:'%v'", resp.StatusCode, string(webhookBody))
+				log.Error().Str("webhook", r.WebhookURL).Msgf("failed to send test result to webhook, status: %d, msg: %q.", resp.StatusCode, string(webhookBody))
 			}
 			if resp.StatusCode%100 == 2 {
-				log.Info().Str("webhook", r.WebhookURL).Msgf("test result has been sent successfully to webhook, msg: '%v'.", string(webhookBody))
+				log.Info().Str("webhook", r.WebhookURL).Msgf("test result has been sent successfully to webhook, msg: %q.", string(webhookBody))
 			}
 		}
 	}
 
 	if r.Filename != "" {
 		err = os.WriteFile(r.Filename, body, 0666)
 		if err != nil {
-			log.Error().Err(err).Msgf("failed to write test result to %s", r.Filename)
+			log.Err(err).Msgf("failed to write test result to %q.", r.Filename)
 		}
 	}
 }
 
-// cleanup removes any information that isn't relevant in the rendered report. Particularly when it comes to
-// artifacts, this reporter is only interested in those that have been persisted to the file system.
-func (r *Reporter) cleanup() {
-	for i, result := range r.Results {
-		var artifacts []report.Artifact
-		for _, a := range result.Artifacts {
-			if a.FilePath == "" {
-				continue
-			}
-			artifacts = append(artifacts, a)
-		}
-		r.Results[i].Artifacts = artifacts
-	}
-}
-
 // Reset resets the reporter to its initial state. This action will delete all test results.
 func (r *Reporter) Reset() {
 	r.Results = make([]report.TestResult, 0)

diff --git a/internal/report/report.go b/internal/report/report.go
@@ -32,10 +32,11 @@ type TestResult struct {
 	Browser       string        `json:"browser,omitempty"`
 	Platform      string        `json:"platform"`
 	DeviceName    string        `json:"deviceName,omitempty"`
-	URL           string        `json:"url"`
+	URL           string        `json:"url,omitempty"`
 	Artifacts     []Artifact    `json:"artifacts,omitempty"`
-	Origin        string        `json:"origin"`
+	Origin        string        `json:"origin,omitempty"`
 	BuildURL      string        `json:"buildURL,omitempty"`
+	RunID         string        `json:"runID,omitempty"`
 	RDC           bool          `json:"-"`
 	TimedOut      bool          `json:"-"`
 	PassThreshold bool          `json:"-"`

diff --git a/internal/saucecloud/imagerunner.go b/internal/saucecloud/imagerunner.go
@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"reflect"
 	"time"
 
@@ -318,7 +319,11 @@ func (r *ImgRunner) collectResults(results chan execResult, expected int) bool {
 
 		r.PrintResult(res)
 		r.PrintLogs(res.runID, res.name)
-		r.DownloadArtifacts(res.runID, res.name, res.status, res.err != nil)
+		files := r.DownloadArtifacts(res.runID, res.name, res.status, res.err != nil)
+		var artifacts []report.Artifact
+		for _, f := range files {
+			artifacts = append(artifacts, report.Artifact{FilePath: f})
+		}
 
 		for _, r := range r.Reporters {
 			r.Add(report.TestResult{
@@ -327,6 +332,9 @@ func (r *ImgRunner) collectResults(results chan execResult, expected int) bool {
 				StartTime: res.startTime,
 				EndTime:   res.endTime,
 				Status:    res.status,
+				Artifacts: artifacts,
+				Platform:  "Linux",
+				RunID:     res.runID,
 				Attempts: []report.Attempt{{
 					ID:        res.runID,
 					Duration:  res.duration,
@@ -391,51 +399,56 @@ func (r *ImgRunner) PollRun(ctx context.Context, id string, lastStatus string) (
 	}
 }
 
-func (r *ImgRunner) DownloadArtifacts(runnerID, suiteName, status string, passed bool) {
+// DownloadArtifact downloads a zipped archive of artifacts and extracts the required files.
+func (r *ImgRunner) DownloadArtifacts(runnerID, suiteName, status string, passed bool) []string {
 	if r.Async ||
 		runnerID == "" ||
 		status == imagerunner.StateCancelled ||
 		!r.Project.Artifacts.Download.When.IsNow(passed) {
-		return
+		return nil
 	}
 
 	dir, err := config.GetSuiteArtifactFolder(suiteName, r.Project.Artifacts.Download)
 	if err != nil {
 		log.Err(err).Msg("Unable to create artifacts folder.")
-		return
+		return nil
 	}
 
 	log.Info().Msg("Downloading artifacts archive")
 	reader, err := r.RunnerService.DownloadArtifacts(r.ctx, runnerID)
 	if err != nil {
 		log.Err(err).Str("suite", suiteName).Msg("Failed to fetch artifacts.")
-		return
+		return nil
 	}
 	defer reader.Close()
 
 	fileName, err := fileio.CreateTemp(reader)
 	if err != nil {
 		log.Err(err).Str("suite", suiteName).Msg("Failed to download artifacts content.")
-		return
+		return nil
 	}
 	defer os.Remove(fileName)
 
 	zf, err := zip.OpenReader(fileName)
 	if err != nil {
-		log.Error().Msgf("Unable to open zip file %s: %s", fileName, err)
-		return
+		log.Err(err).Msgf("Unable to open zip file %q", fileName)
+		return nil
 	}
 	defer zf.Close()
+	var artifacts []string
 	for _, f := range zf.File {
 		for _, pattern := range r.Project.Artifacts.Download.Match {
 			if glob.Glob(pattern, f.Name) {
 				if err = szip.Extract(dir, f); err != nil {
-					log.Error().Msgf("Unable to extract file '%s': %s", f.Name, err)
+					log.Err(err).Msgf("Unable to extract file %q", f.Name)
+				} else {
+					artifacts = append(artifacts, filepath.Join(dir, f.Name))
 				}
 				break
 			}
 		}
 	}
+	return artifacts
 }
 
 func (r *ImgRunner) PrintResult(res execResult) {