validation: Use prove(1) as a TAP harness

Capture stdout and stderr from create invocations, because we don't
want to pollute the TAP output with things like runc's:

  Incorrect Usage.
  ...

(which is for some reason printed to stdout) or:

  runc: "create" requires exactly 1 argument(s)

which is printed to stderr.  Instead, show the captured stderr as a
diagnostic, and hide the stdout completely.  Unless stderr is empty,
in which case show stdout in case it contains something useful.

Most of these tests are broken because we aren't collecting the
container exit code or post-start stdout.  But the tests haven't been
doing that since the create/start split in 15577bd (add runtime
struct; add create test, 2017-08-24, opencontainers#447) anyway [1].  This commit
just makes that more obvious.

The patsubst and wildcard Makefile syntax is documented in [2].  The
$(VALIDATION_TESTS) rule uses the static pattern rule syntax [3].

[1]: opencontainers#447 (comment)
[2]: https://www.gnu.org/software/make/manual/html_node/Wildcard-Function.html
[3]: https://www.gnu.org/software/make/manual/html_node/Static-Usage.html#Static-Usage

Signed-off-by: W. Trevor King <wking@tremily.us>

Makefile

@@ -5,6 +5,7 @@ BUILDTAGS=
 RUNTIME ?= runc
 COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 VERSION := ${shell cat ./VERSION}
+VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))
 
 all: tool runtimetest
 
@@ -35,10 +36,14 @@ uninstall:
 	rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool
 
 clean:
-	rm -f oci-runtime-tool runtimetest *.1
+	rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
 
-localvalidation: runtimetest
-	RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
+localvalidation: runtimetest $(VALIDATION_TESTS)
+	RUNTIME=$(RUNTIME) prove $(VALIDATION_TESTS)
+
+.PHONY: $(VALIDATION_TESTS)
+$(VALIDATION_TESTS): %.t: %.go
+	go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
 
 .PHONY: test .gofmt .govet .golint
 

validation/.gitignore

@@ -0,0 +1 @@
+/*.t

validation/create.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mndrix/tap-go"
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	"github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+
+	g := generate.New()
+	g.SetRootPath(".")
+	g.SetProcessArgs([]string{"ls"})
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	r, err := util.NewRuntime(util.RuntimeCommand, bundleDir)
+	if err != nil {
+		util.Fatal(err)
+	}
+	defer r.Clean(true)
+
+	err = r.SetConfig(&g)
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	containerID := uuid.NewV4().String()
+	cases := []struct {
+		id          string
+		errExpected bool
+		err         error
+	}{
+		{"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)},
+		{containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)},
+		{containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)},
+	}
+
+	for _, c := range cases {
+		r.SetID(c.id)
+		stderr, err := r.Create()
+		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
+		t.Diagnostic(c.err.(*specerror.Error).Err.Reference)
+		t.Diagnostic(err.Error())
+		if len(stderr) > 0 {
+			t.Diagnostic(string(stderr))
+		}
+
+		if err == nil {
+			state, _ := r.State()
+			t.Ok(state.ID == c.id, "")
+			t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID)
+		}
+	}
+
+	t.AutoPlan()
+}

validation/default.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/hostname.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetHostname("hostname-specific")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_devices.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"os"
+
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+
+	// add char device
+	cdev := rspecs.LinuxDevice{}
+	cdev.Path = "/dev/test1"
+	cdev.Type = "c"
+	cdev.Major = 10
+	cdev.Minor = 666
+	cmode := os.FileMode(int32(432))
+	cdev.FileMode = &cmode
+	cuid := uint32(0)
+	cdev.UID = &cuid
+	cgid := uint32(0)
+	cdev.GID = &cgid
+	g.AddDevice(cdev)
+
+	// add block device
+	bdev := rspecs.LinuxDevice{}
+	bdev.Path = "/dev/test2"
+	bdev.Type = "b"
+	bdev.Major = 8
+	bdev.Minor = 666
+	bmode := os.FileMode(int32(432))
+	bdev.FileMode = &bmode
+	uid := uint32(0)
+	bdev.UID = &uid
+	gid := uint32(0)
+	bdev.GID = &gid
+	g.AddDevice(bdev)
+
+	// add fifo device
+	pdev := rspecs.LinuxDevice{}
+	pdev.Path = "/dev/test3"
+	pdev.Type = "p"
+	pdev.Major = 8
+	pdev.Minor = 666
+	pmode := os.FileMode(int32(432))
+	pdev.FileMode = &pmode
+	g.AddDevice(pdev)
+
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_gid_mappings.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200))
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_masked_paths.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxMaskedPaths("/masktest")
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "masktest")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_readonly_paths.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxReadonlyPaths("readonlytest")
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "readonlytest")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_rootfs_propagation_shared.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetupPrivileged(true)
+	g.SetLinuxRootPropagation("shared")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_rootfs_propagation_unbindable.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetupPrivileged(true)
+	g.SetLinuxRootPropagation("unbindable")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_sysctl.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_uid_mappings.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(3200))
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/mounts.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	util.Skip("TODO: mounts generation options have not been implemented", "")
+}

validation/process.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetProcessCwd("/test")
+	g.AddProcessEnv("testa", "valuea")
+	g.AddProcessEnv("testb", "123")
+
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "test")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/process_capabilities.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	if "linux" != runtime.GOOS {
+		util.Skip("linux-specific process.capabilities test", runtime.GOOS)
+		os.Exit(0)
+	}
+
+	g := util.GetDefaultGenerator()
+	g.SetupPrivileged(true)
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/process_oom_score_adj.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetProcessOOMScoreAdj(500)
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/process_rlimits.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddProcessRlimits("RLIMIT_NOFILE", 1024, 1024)
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}