Add sharding for XCUITest (#772)

* Add sharding for XCUITest * rename function * rename function * Update internal/xcuitest/config.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * Update internal/xcuitest/config.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * update according to comments * update description for testClassesFile * update comment for binpack * rename testing function * update function name * update error msg * refine test description * add test helper * remove all shardConfig * Update description * update JSON schema * fix typo * Update shard JSON schema * fail fast * rename parseTestClassesFile to getShardedSuites * use t.TempDir() * Update internal/xcuitest/config.go Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com> * remove useless check * fix ut * rename to TestListFile * add a test case for empty lines in testListFile * update description for testListFile * refine function comment * refine trim space * refine checks for testListFile * refine * relocate file.Close() --------- Co-authored-by: Alex Plischke <alex.plischke@saucelabs.com>
saucelabs · May 15, 2023 · 244338f · 244338f
1 parent 438d710
commit 244338f
Show file tree

Hide file tree

Showing 12 changed files with 289 additions and 16 deletions.
diff --git a/api/saucectl.schema.json b/api/saucectl.schema.json
@@ -2131,6 +2131,17 @@
                 },
                 "smartRetry": {
                   "$ref": "#/allOf/0/then/properties/suites/items/properties/smartRetry"
+                },
+                "shard": {
+                  "description": "When shard is configured as concurrency, saucectl automatically splits the tests by concurrency so that they can easily run in parallel.",
+                  "enum": [
+                    "",
+                    "concurrency"
+                  ]
+                },
+                "testListFile": {
+                  "description": "This file containing tests will be used in sharding by concurrency.",
+                  "type": "string"
                 }
               },
               "anyOf": [

diff --git a/api/v1alpha/framework/xcuitest.schema.json b/api/v1alpha/framework/xcuitest.schema.json
@@ -174,6 +174,17 @@
           },
           "smartRetry": {
             "$ref": "../subschema/common.schema.json#/definitions/smartRetry"
+          },
+          "shard": {
+            "description": "When shard is configured as concurrency, saucectl automatically splits the tests by concurrency so that they can easily run in parallel.",
+            "enum": [
+              "",
+              "concurrency"
+            ]
+          },
+          "testListFile": {
+            "description": "This file containing tests will be used in sharding by concurrency.",
+            "type": "string"
           }
         },
         "anyOf": [

diff --git a/internal/cmd/run/xcuitest.go b/internal/cmd/run/xcuitest.go
@@ -68,6 +68,9 @@ func NewXCUITestCmd() *cobra.Command {
 	sc.StringSlice("otherApps", "xcuitest::otherApps", []string{}, "Specifies any additional apps that are installed alongside the main app")
 	sc.Int("passThreshold", "suite::passThreshold", 1, "The minimum number of successful attempts for a suite to be considered as 'passed'.")
 
+	sc.String("shard", "suite::shard", "", "When shard is configured as concurrency, saucectl automatically splits the tests by concurrency so that they can easily run in parallel. Requires --name to be set.")
+	sc.String("testListFile", "suite::testListFile", "", "This file containing tests will be used in sharding by concurrency. Requires --name to be set.")
+
 	// Test Options
 	sc.StringSlice("testOptions.class", "suite::testOptions::class", []string{}, "Only run the specified classes. Requires --name to be set.")
 	sc.StringSlice("testOptions.notClass", "suite::testOptions::notClass", []string{}, "Run all classes except those specified here. Requires --name to be set.")
@@ -102,6 +105,9 @@ func runXcuitest(cmd *cobra.Command, xcuiFlags xcuitestFlags, isCLIDriven bool)
 	if err := xcuitest.Validate(p); err != nil {
 		return 1, err
 	}
+	if err := xcuitest.ShardSuites(&p); err != nil {
+		return 1, err
+	}
 
 	regio := region.FromString(p.Sauce.Region)
 
@@ -125,7 +131,8 @@ func runXcuitest(cmd *cobra.Command, xcuiFlags xcuitestFlags, isCLIDriven bool)
 	go func() {
 		props := usage.Properties{}
 		props.SetFramework("xcuitest").SetFlags(cmd.Flags()).SetSauceConfig(p.Sauce).SetArtifacts(p.Artifacts).
-			SetNumSuites(len(p.Suites)).SetJobs(captor.Default.TestResults).SetSlack(p.Notifications.Slack).SetLaunchOrder(p.Sauce.LaunchOrder)
+			SetNumSuites(len(p.Suites)).SetJobs(captor.Default.TestResults).SetSlack(p.Notifications.Slack).
+			SetSharding(xcuitest.IsSharded(p.Suites)).SetLaunchOrder(p.Sauce.LaunchOrder)
 		tracker.Collect(cases.Title(language.English).String(cmds.FullName(cmd)), props)
 		_ = tracker.Close()
 	}()

diff --git a/internal/concurrency/concurrency.go b/internal/concurrency/concurrency.go
@@ -1,16 +1,16 @@
 package concurrency
 
-// SplitTestFiles splits test files into groups to match concurrency
-func SplitTestFiles(files []string, concurrency int) [][]string {
+// BinPack splits items into groups to match concurrency
+func BinPack(items []string, concurrency int) [][]string {
 	if concurrency == 1 {
-		return [][]string{files}
+		return [][]string{items}
 	}
-	if concurrency > len(files) {
-		concurrency = len(files)
+	if concurrency > len(items) {
+		concurrency = len(items)
 	}
 	buckets := make([][]string, concurrency)
-	for i := 0; i < len(files); i++ {
-		buckets[i%concurrency] = append(buckets[i%concurrency], files[i])
+	for i := 0; i < len(items); i++ {
+		buckets[i%concurrency] = append(buckets[i%concurrency], items[i])
 	}
 
 	return buckets

diff --git a/internal/concurrency/concurrency_test.go b/internal/concurrency/concurrency_test.go
@@ -6,7 +6,7 @@ import (
 	"gotest.tools/v3/assert"
 )
 
-func Test_SplitTestFiles(t *testing.T) {
+func Test_BinPack(t *testing.T) {
 	var testCases = []struct {
 		name      string
 		files     []string
@@ -41,7 +41,7 @@ func Test_SplitTestFiles(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			result := SplitTestFiles(tc.files, tc.count)
+			result := BinPack(tc.files, tc.count)
 			assert.Equal(t, len(tc.expResult), len(result))
 			for i := 0; i < len(result); i++ {
 				for j := 0; j < len(result[i]); j++ {

diff --git a/internal/cucumber/config.go b/internal/cucumber/config.go
@@ -245,7 +245,7 @@ func shardSuites(rootDir string, suites []Suite, ccy int) ([]Suite, error) {
 			}
 		}
 		if s.Shard == "concurrency" {
-			groups := concurrency.SplitTestFiles(testFiles, ccy)
+			groups := concurrency.BinPack(testFiles, ccy)
 			for i, group := range groups {
 				replica := s
 				replica.Name = fmt.Sprintf("%s - %d/%d", s.Name, i+1, len(groups))

diff --git a/internal/cypress/v1/config.go b/internal/cypress/v1/config.go
@@ -309,7 +309,7 @@ func shardSuites(rootDir string, suites []Suite, ccy int) ([]Suite, error) {
 			}
 		}
 		if s.Shard == "concurrency" {
-			fileGroups := concurrency.SplitTestFiles(testFiles, ccy)
+			fileGroups := concurrency.BinPack(testFiles, ccy)
 			for i, group := range fileGroups {
 				replica := s
 				replica.Name = fmt.Sprintf("%s - %d/%d", s.Name, i+1, len(fileGroups))

diff --git a/internal/cypress/v1alpha/config.go b/internal/cypress/v1alpha/config.go
@@ -358,7 +358,7 @@ func shardSuites(cfg Config, suites []Suite, ccy int) ([]Suite, error) {
 			}
 		}
 		if s.Shard == "concurrency" {
-			fileGroups := concurrency.SplitTestFiles(testFiles, ccy)
+			fileGroups := concurrency.BinPack(testFiles, ccy)
 			for i, group := range fileGroups {
 				replica := s
 				replica.Name = fmt.Sprintf("%s - %d/%d", s.Name, i+1, len(fileGroups))

diff --git a/internal/playwright/config.go b/internal/playwright/config.go
@@ -234,7 +234,7 @@ func shardInSuites(rootDir string, suites []Suite, ccy int) ([]Suite, error) {
 			}
 		}
 		if s.Shard == "concurrency" {
-			groups := concurrency.SplitTestFiles(testFiles, ccy)
+			groups := concurrency.BinPack(testFiles, ccy)
 			for i, group := range groups {
 				replica := s
 				replica.Name = fmt.Sprintf("%s - %d/%d", s.Name, i+1, len(groups))

diff --git a/internal/testcafe/config.go b/internal/testcafe/config.go
@@ -344,7 +344,7 @@ func shardSuites(rootDir string, suites []Suite, ccy int) ([]Suite, error) {
 			}
 		}
 		if s.Shard == "concurrency" {
-			groups := concurrency.SplitTestFiles(testFiles, ccy)
+			groups := concurrency.BinPack(testFiles, ccy)
 			for i, group := range groups {
 				replica := s
 				replica.Name = fmt.Sprintf("%s - %d/%d", s.Name, i+1, len(groups))

diff --git a/internal/xcuitest/config.go b/internal/xcuitest/config.go
@@ -1,13 +1,16 @@
 package xcuitest
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
 	"github.com/saucelabs/saucectl/internal/apps"
+	"github.com/saucelabs/saucectl/internal/concurrency"
 	"github.com/saucelabs/saucectl/internal/config"
 	"github.com/saucelabs/saucectl/internal/insights"
 	"github.com/saucelabs/saucectl/internal/msg"
@@ -67,6 +70,8 @@ type Suite struct {
 	AppSettings        config.AppSettings `yaml:"appSettings,omitempty" json:"appSettings"`
 	PassThreshold      int                `yaml:"passThreshold,omitempty" json:"-"`
 	SmartRetry         config.SmartRetry  `yaml:"smartRetry,omitempty" json:"-"`
+	Shard              string             `yaml:"shard,omitempty" json:"-"`
+	TestListFile       string             `yaml:"testListFile,omitempty" json:"-"`
 }
 
 // IOS constant
@@ -226,3 +231,63 @@ func SortByHistory(suites []Suite, history insights.JobHistory) []Suite {
 	}
 	return res
 }
+
+// ShardSuites applies sharding by provided testListFile.
+func ShardSuites(p *Project) error {
+	var suites []Suite
+	for _, s := range p.Suites {
+		if s.Shard != "concurrency" {
+			suites = append(suites, s)
+			continue
+		}
+		shardedSuites, err := getShardedSuites(s, p.Sauce.Concurrency)
+		if err != nil {
+			return fmt.Errorf("failed to get tests from testListFile(%q): %v", s.TestListFile, err)
+		}
+		suites = append(suites, shardedSuites...)
+	}
+	p.Suites = suites
+
+	return nil
+}
+
+func getShardedSuites(suite Suite, ccy int) ([]Suite, error) {
+	readFile, err := os.Open(suite.TestListFile)
+	if err != nil {
+		return nil, err
+	}
+	defer readFile.Close()
+
+	fileScanner := bufio.NewScanner(readFile)
+	fileScanner.Split(bufio.ScanLines)
+	var tests []string
+	for fileScanner.Scan() {
+		text := strings.TrimSpace(fileScanner.Text())
+		if text == "" {
+			continue
+		}
+		tests = append(tests, text)
+	}
+	if len(tests) == 0 {
+		return nil, errors.New("empty file")
+	}
+
+	buckets := concurrency.BinPack(tests, ccy)
+	var suites []Suite
+	for i, b := range buckets {
+		currSuite := suite
+		currSuite.Name = fmt.Sprintf("%s - %d/%d", suite.Name, i+1, ccy)
+		currSuite.TestOptions.Class = b
+		suites = append(suites, currSuite)
+	}
+	return suites, nil
+}
+
+func IsSharded(suites []Suite) bool {
+	for _, s := range suites {
+		if s.Shard != "" {
+			return true
+		}
+	}
+	return false
+}