mantle: Add argument to enable channel based testing

This commit adds the argument `--channel` to enable channel based testing, which
means you can populate the Channel property in kola tests and kola would decide to
include/exclude the test based on which channel it is running

Fixes flatcar/Flatcar#42

Signed-off-by: Sayan Chowdhury <sayan@kinvolk.io>

cmd/kola/kola.go

@@ -130,7 +130,7 @@ func runRun(cmd *cobra.Command, args []string) {
 	} else {
 		sshKeys = nil
 	}
-	runErr := kola.RunTests(patterns, kolaPlatform, outputDir, &sshKeys, runRemove)
+	runErr := kola.RunTests(patterns, kolaChannel, kolaPlatform, outputDir, &sshKeys, runRemove)
 
 	// needs to be after RunTests() because harness empties the directory
 	if err := writeProps(); err != nil {
@@ -278,7 +278,7 @@ func runList(cmd *cobra.Command, args []string) {
 			patterns = []string{"*"} // run all tests by default
 		}
 		var err error
-		tests, err = kola.FilterTests(register.Tests, patterns, kolaPlatform, semver.Version{})
+		tests, err = kola.FilterTests(register.Tests, patterns, kolaChannel, kolaPlatform, semver.Version{})
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "filtering error: %v\n", err)
 			os.Exit(1)

cmd/kola/options.go

@@ -35,10 +35,12 @@ import (
 var (
 	outputDir          string
 	kolaPlatform       string
+	kolaChannel        string
 	defaultTargetBoard = sdk.DefaultBoard()
 	kolaArchitectures  = []string{"amd64"}
 	kolaPlatforms      = []string{"aws", "azure", "do", "esx", "gce", "openstack", "packet", "qemu", "qemu-unpriv"}
 	kolaDistros        = []string{"cl", "fcos", "rhcos"}
+	kolaChannels       = []string{"alpha", "beta", "stable", "edge"}
 	kolaDefaultImages  = map[string]string{
 		"amd64-usr": sdk.BuildRoot() + "/images/amd64-usr/latest/flatcar_production_image.bin",
 		"arm64-usr": sdk.BuildRoot() + "/images/arm64-usr/latest/flatcar_production_image.bin",
@@ -64,6 +66,7 @@ func init() {
 	sv(&outputDir, "output-dir", "", "Temporary output directory for test data and logs")
 	sv(&kola.TorcxManifestFile, "torcx-manifest", "", "Path to a torcx manifest that should be made available to tests")
 	root.PersistentFlags().StringVarP(&kolaPlatform, "platform", "p", "qemu", "VM platform: "+strings.Join(kolaPlatforms, ", "))
+	root.PersistentFlags().StringVarP(&kolaChannel, "channel", "c", "stable", "Channel: "+strings.Join(kolaChannels, ", "))
 	root.PersistentFlags().StringVarP(&kola.Options.Distribution, "distro", "b", "cl", "Distribution: "+strings.Join(kolaDistros, ", "))
 	root.PersistentFlags().IntVarP(&kola.TestParallelism, "parallel", "j", 1, "number of tests to run in parallel")
 	sv(&kola.TAPFile, "tapfile", "", "file to write TAP results to")
@@ -171,6 +174,10 @@ func syncOptions() error {
 		return err
 	}
 
+	if err := validateOption("channel", kolaChannel, kolaChannels); err != nil {
+		return err
+	}
+
 	if err := validateOption("distro", kola.Options.Distribution, kolaDistros); err != nil {
 		return err
 	}

kola/harness.go

@@ -188,7 +188,7 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) {
 	return
 }
 
-func FilterTests(tests map[string]*register.Test, patterns []string, pltfrm string, version semver.Version) (map[string]*register.Test, error) {
+func FilterTests(tests map[string]*register.Test, patterns []string, channel, pltfrm string, version semver.Version) (map[string]*register.Test, error) {
 	r := make(map[string]*register.Test)
 
 	checkPlatforms := []string{pltfrm}
@@ -278,6 +278,10 @@ func FilterTests(tests map[string]*register.Test, patterns []string, pltfrm stri
 			continue
 		}
 
+		if allowed, excluded := isAllowed(channel, t.Channels, t.ExcludeChannels); !allowed || excluded {
+			continue
+		}
+
 		r[name] = t
 	}
 
@@ -309,7 +313,7 @@ func versionOutsideRange(version, minVersion, endVersion semver.Version) bool {
 // register tests in their init() function.
 // outputDir is where various test logs and data will be written for
 // analysis after the test run. If it already exists it will be erased!
-func RunTests(patterns []string, pltfrm, outputDir string, sshKeys *[]agent.Key, remove bool) error {
+func RunTests(patterns []string, channel, pltfrm, outputDir string, sshKeys *[]agent.Key, remove bool) error {
 	var versionStr string
 
 	// Avoid incurring cost of starting machine in getClusterSemver when
@@ -318,7 +322,7 @@ func RunTests(patterns []string, pltfrm, outputDir string, sshKeys *[]agent.Key,
 	// 2) glob is an exact match which means minVersion will be ignored
 	//    either way
 	// 3) the provided torcx flag is wrong
-	tests, err := FilterTests(register.Tests, patterns, pltfrm, semver.Version{})
+	tests, err := FilterTests(register.Tests, patterns, channel, pltfrm, semver.Version{})
 	if err != nil {
 		plog.Fatal(err)
 	}
@@ -369,7 +373,7 @@ func RunTests(patterns []string, pltfrm, outputDir string, sshKeys *[]agent.Key,
 		versionStr = version.String()
 
 		// one more filter pass now that we know real version
-		tests, err = FilterTests(tests, patterns, pltfrm, *version)
+		tests, err = FilterTests(tests, patterns, channel, pltfrm, *version)
 		if err != nil {
 			plog.Fatal(err)
 		}

kola/register/register.go

@@ -56,6 +56,8 @@ type Test struct {
 	ExcludePlatforms []string // blacklist of platforms to ignore -- defaults to none
 	Distros          []string // whitelist of distributions to run test against -- defaults to all
 	ExcludeDistros   []string // blacklist of distributions to ignore -- defaults to none
+	Channels         []string // whitelist of channels to run test against -- defaults to all
+	ExcludeChannels  []string // blacklist of channels to ignore -- defaults to all
 	Architectures    []string // whitelist of machine architectures supported -- defaults to all
 	Flags            []Flag   // special-case options for this test