kola: Replace run-ext-bin with kola run -E/--exttest

mantle/cmd/kola/kola.go

@@ -84,22 +84,6 @@ will be ignored.
 		SilenceUsage: true,
 	}
 
-	cmdRunExtBin = &cobra.Command{
-		Use:   "run-ext-bin",
-		Short: "Run an external test (single binary)",
-		Long: `Run an externally defined test
-
-This injects a single binary into the target system and executes it as a systemd
-unit.  The test is considered successful when the service exits normally, and
-failed if the test exits non-zero - but the service being killed by e.g. SIGTERM
-is ignored.  This is intended to allow rebooting the system.
-`,
-
-		Args:    cobra.ExactArgs(1),
-		PreRunE: preRun,
-		RunE:    runRunExtBin,
-	}
-
 	cmdHttpServer = &cobra.Command{
 		Use:   "http-server",
 		Short: "Run a static webserver",
@@ -140,14 +124,15 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
 	qemuImageDirIsTemp bool
 
 	extDependencyDir string
+	runExternals     []string
 )
 
 func init() {
 	root.AddCommand(cmdRun)
-	root.AddCommand(cmdRunExtBin)
-	cmdRunExtBin.Flags().StringVar(&extDependencyDir, "depdir", "", "Copy (rsync) dir to target, available as ${KOLA_EXT_DATA}")
+	cmdRun.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests (will be found in DIR/tests/kola)")
 
 	root.AddCommand(cmdList)
+	cmdList.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests in directory")
 	cmdList.Flags().BoolVar(&listJSON, "json", false, "format output in JSON")
 	cmdList.Flags().StringVarP(&listPlatform, "platform", "p", "all", "filter output by platform")
 	cmdList.Flags().StringVarP(&listDistro, "distro", "b", "all", "filter output by distro")
@@ -200,6 +185,13 @@ func runRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	for _, d := range runExternals {
+		err := kola.RegisterExternalTests(d)
+		if err != nil {
+			return err
+		}
+	}
+
 	runErr := kola.RunTests(patterns, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
 
 	// needs to be after RunTests() because harness empties the directory
@@ -210,22 +202,6 @@ func runRun(cmd *cobra.Command, args []string) error {
 	return runErr
 }
 
-func runRunExtBin(cmd *cobra.Command, args []string) error {
-	extbin := args[0]
-
-	outputDir, err := kola.SetupOutputDir(outputDir, kolaPlatform)
-	if err != nil {
-		return err
-	}
-
-	runErr := kola.RunExtBin(kolaPlatform, outputDir, extbin, extDependencyDir)
-	if err := writeProps(); err != nil {
-		return errors.Wrapf(err, "writing properties")
-	}
-
-	return runErr
-}
-
 func writeProps() error {
 	f, err := os.OpenFile(filepath.Join(outputDir, "properties.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
 	if err != nil {
@@ -348,6 +324,12 @@ func writeProps() error {
 }
 
 func runList(cmd *cobra.Command, args []string) error {
+	for _, d := range runExternals {
+		err := kola.RegisterExternalTests(d)
+		if err != nil {
+			return err
+		}
+	}
 	var testlist []*item
 	for name, test := range register.Tests {
 		item := &item{

mantle/cmd/kolet/kolet.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"syscall"
 	"time"
 
 	systemddbus "github.com/coreos/go-systemd/v22/dbus"
@@ -138,9 +139,16 @@ func dispatchRunExtUnit(unitname string, sdconn *systemddbus.Conn) (bool, error)
 					}
 				case CLD_KILLED:
 					// SIGTERM; we explicitly allow that, expecting we're rebooting.
-					if mainstatus == 15 {
+					if mainstatus == int32(15) {
 						fmt.Printf("Unit %s terminated via SIGTERM, assuming reboot\n", unitname)
-						return false, nil
+						// Now we send ourself SIGTERM, so that that propagates back to the ssh
+						// status, so that the kola runner can use that as a trigger to reboot.
+						selfproc := os.Process{
+							Pid: os.Getpid(),
+						}
+						selfproc.Signal(syscall.SIGTERM)
+						time.Sleep(time.Hour)
+						panic("failed to send SIGTERM to self")
 					} else {
 						return true, fmt.Errorf("Unit %s killed by signal %d", unitname, mainstatus)
 					}
@@ -157,7 +165,7 @@ func dispatchRunExtUnit(unitname string, sdconn *systemddbus.Conn) (bool, error)
 			}
 		}
 	default:
-		return false, fmt.Errorf("Unhandled systemd unit state %s", state)
+		return false, fmt.Errorf("Unhandled systemd unit state:%s", state)
 	}
 }
 

mantle/kola/README-kola-ext.md

@@ -1,5 +1,8 @@
 Integrating external test suites
-===
+=
+
+Rationale
+---
 
 Fedora CoreOS is comprised of a number of upstream projects, from
 the Linux kernel to systemd, ostree, Ignition, podman and
@@ -10,31 +13,83 @@ repositories, and allow these projects to target Fedora CoreOS
 in e.g. their own CI.  And we also want to run unmodified
 upstream tests, *without rebuilding* the project.
 
-Using kola run-ext-bin
-===
+Using kola run -E/--exttest
+---
+
+The `--exttest` (`-E`) argument to `kola run` one way to accomplish this; you
+provide the path to an upstream project git repository.  Tests will be found
+in the `tests/kola` directory.  If this project contains binaries that require
+building, it is assumed that `make` (or equivalent) has already been invoked.
+
+The `tests/kola` directory will be traversed recursively to find tests.
+
+The core idea is to express a test as a single binary (plus an optional
+directory of dependencies named `data`) that will be uploaded to a CoreOS
+system and run as a systemd unit, along with an optional Ignition config
+named `config.ign`.
 
-The `kola run-ext-bin` is one way to accomplish this.  The
-core idea is to express your test suite as a binary (plus an optional
-directory of dependencies) that will be uploaded to a CoreOS
-system and run as a systemd unit.
+Concretely then, an external test directory can have the following content:
 
-Currently this systemd unit runs with full privileges - tests
+- `config.ign` (optional): Ignition config provided
+- `kola.json` (optional): JSON file described below
+- `data` (optional): Directory (or symlink to dir): Will be uploaded and available as `${KOLA_EXT_DATA}`
+- one or more executables: Each executable is its own test, run independently
+  with the Ignition config and/or dependency data provided.
+
+In the case of a test directory with a single executable, the kola test name will be
+`ext.<projname>.<subdirectory>`.  Otherwise, the test will be named `ext.<projname>.<subdirectory>.<executable>`.
+
+Currently the test systemd unit runs with full privileges - tests
 are assumed to be (potentially) destructive and a general assumption
-is tests are run in easily disposable virtual machines.
+is tests are run in easily disposable virtual machines.  A future
+enhancement will support nondestructive tests.
 
 A best practice for doing this is to write your tests in a language
 that compiles to a single binary - Rust and Go for example, but
 there exist for example tools like [PyInstaller](https://realpython.com/pyinstaller-python/#pyinstaller)
-too.
+too.  This way you can usually avoid the need for a `data` directory.
 
 This mechanism is suitable for testing most userspace components
 of CoreOS; for example, one can have the binary drive a container runtime.
 
-An important feature of `run-ext-bin` is support for rebooting the host system.
-This allows one to easily test OS updates.  To do this, simply invoke the usual
-`reboot` - the test framework will monitor the target systemd unit
-and ignore the case where it exits with `SIGTERM`.
-
 A test is considered failed if the unit exits with any non-zero exit
 status or dies from any signal other than `SIGTERM`.
 
+Support for rebooting
+---
+
+An important feature of exttests is support for rebooting the host system.
+This allows one to easily test OS updates for example.  To do this, your
+test process should send `SIGTERM` to itself.  For example, in bash use:
+
+`kill -TERM $$`
+
+This will trigger the monitoring `kola` process to invoke a reboot.
+
+The rationale for this is that it helps kola to know when a reboot
+is happening so that it can correctly follow the state of the systemd
+journal, etc.  A future enhancement will support directly invoking
+`reboot` and having kola just figure it out.
+
+`kola.json`
+---
+
+Kola internally supports limiting tests to specific architectures and plaforms,
+as well as "clusters" of machines that have size > 1.  External tests
+are hardcoded to 1 machine at the moment.
+
+Here's an example `kola.json`:
+
+```json
+{
+    "architectures": "!s390x ppc64le",
+    "platforms": "qemu-unpriv"
+}
+```
+
+The only supported keys are those two; either or none may be provided as well.
+Each value is a single string, which is a whitespace-separated list.
+The reason to use a single string (instead of a native JSON list)
+is that by providing `!` at the front of the string, the value instead
+declares exclusions (`ExclusiveArchitectures` instead of `Architectures` in
+reference to kola internals.