[DRAFT - WIP] Add example for CustomTest command

examples/custom-tests/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.15 as builder
+COPY main.go .
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
+
+FROM alpine:3
+# Define GOTRACEBACK to mark this container as using the Go language runtime
+# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
+ENV GOTRACEBACK=single
+CMD ["./app"]
+COPY --from=builder /app .

examples/custom-tests/README.md

@@ -0,0 +1,25 @@
+### Example: Running custom tests on built images
+
+This example shows how to run
+[custom tests]
+on newly built images in your skaffold dev loop. Tests are associated with single
+artifacts. Tests are configured in
+your `skaffold.yaml` in the `test` stanza, e.g.
+
+```yaml
+test:
+    - image: skaffold-example
+    Custom:
+        - command: <command>
+```
+
+Tests can also be configured through profiles, e.g.
+
+```yaml
+profiles:
+  - name: test
+    test:
+      - image: skaffold-example
+        Custom:
+        - command: <command>
+```

examples/custom-tests/k8s-pod.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: getting-started
+spec:
+  containers:
+  - name: getting-started
+    image: skaffold-example

examples/custom-tests/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+func main() {
+	for {
+		fmt.Println("Hello world!")
+
+		time.Sleep(time.Second * 1)
+	}
+}

examples/custom-tests/skaffold.yaml

@@ -0,0 +1,31 @@
+apiVersion: skaffold/v2beta12
+kind: Config
+build:
+  artifacts:
+  - image: skaffold-example
+test:
+  - image: skaffold-example
+    custom:
+      - command: ./test/gotest.sh
+        timeout: 300 #in seconds - 5mins
+        dependencies:
+          paths:
+          - "**.go"
+    structureTests:
+      - ./test/*
+deploy:
+  kubectl:
+    manifests:
+      - k8s-*
+profiles:
+  - name: test
+    test:
+      - image: skaffold-example
+        custom:
+          - command: ./test/gotest.sh
+            timeout: 300 #in seconds - 5mins
+            dependencies:
+              paths:
+              - "**.go"
+        structureTests:
+          - ./test/profile_structure_test.yaml

examples/custom-tests/test/gotest.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright 2021 The Skaffold Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# This script runs go test with a better output:
+# - It prints the failures in RED
+# - It recaps the failures at the end
+# - It lists the 20 slowest tests
+
+echo "go test $@"
+
+go run ./test/main.go $@

examples/custom-tests/test/main.go

@@ -0,0 +1,187 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+)
+
+type LogLine struct {
+	Action  string
+	Test    string
+	Package string
+	Output  string
+	Elapsed float32
+}
+
+func main() {
+	if err := goTest(os.Args[1:]); err != nil {
+		os.Exit(1)
+	}
+}
+
+func goTest(testArgs []string) error {
+	args := append([]string{"test", "-json"}, testArgs...)
+	verbose := isVerbose(testArgs)
+
+	cmd := exec.CommandContext(context.Background(), "go", args...)
+
+	pr, pw := io.Pipe()
+	cmd.Stderr = pw
+	cmd.Stdout = pw
+
+	failedTests := map[string]bool{}
+	var failedLogs []LogLine
+	var allLogs []LogLine
+
+	var wc sync.WaitGroup
+	wc.Add(1)
+
+	go func() {
+		defer wc.Done()
+
+		// Print logs while tests are running
+		scanner := bufio.NewScanner(pr)
+		for i := 0; scanner.Scan(); i++ {
+			line := scanner.Bytes()
+
+			var l LogLine
+			if err := json.Unmarshal(line, &l); err != nil {
+				// Sometimes, `go test -json` outputs plain text instead of json.
+				// For example in case of a build error.
+				fmt.Println(failInRed(string(line)))
+				continue
+			}
+
+			allLogs = append(allLogs, l)
+
+			if l.Action == "output" {
+				if verbose || (l.Test == "" && l.Output != "PASS\n" && l.Output != "FAIL\n" && !strings.HasPrefix(l.Output, "coverage:") && !strings.Contains(l.Output, "[no test files]")) {
+					fmt.Print(failInRed(l.Output))
+				}
+			}
+
+			// Is this an error?
+			if (l.Action == "fail" || strings.Contains(l.Output, "FAIL")) && l.Test != "" {
+				if failedTests[l.Package+"/"+l.Test] {
+					continue
+				}
+				failedTests[l.Package+"/"+l.Test] = true
+
+				failedLogs = append(failedLogs, l)
+			}
+		}
+
+		// Print detailed information about failures.
+		if len(failedLogs) > 0 {
+			fmt.Println(red("\n=== Failed Tests ==="))
+
+			for _, l := range failedLogs {
+				fmt.Println(bold(trimPackage(l.Package) + "/" + l.Test))
+
+				for _, l := range logsForTest(l.Test, l.Package, allLogs) {
+					if l.Action == "output" && l.Output != "" && !strings.HasPrefix(l.Output, "=== RUN") {
+						fmt.Print(failInRed(l.Output))
+					}
+				}
+			}
+		}
+
+		// Print top slowest tests.
+		fmt.Println(yellow("\n=== Slow Tests ==="))
+		for _, l := range topSlowest(20, allLogs) {
+			fmt.Printf("%.2fs\t%s\n", l.Elapsed, l.Test)
+		}
+	}()
+
+	err := cmd.Run()
+	if err != nil {
+		pr.CloseWithError(err)
+	} else {
+		pr.Close()
+	}
+	wc.Wait()
+	return err
+}
+
+func failInRed(msg string) string {
+	return strings.ReplaceAll(msg, "FAIL", red("FAIL"))
+}
+
+func red(msg string) string {
+	return "\033[0;31m" + msg + "\033[0m"
+}
+
+func yellow(msg string) string {
+	return "\033[0;33m" + msg + "\033[0m"
+}
+
+func bold(msg string) string {
+	return "\033[1m" + msg + "\033[0m"
+}
+
+func trimPackage(pkg string) string {
+	return strings.TrimPrefix(pkg, "github.com/GoogleContainerTools/skaffold")
+}
+
+func isVerbose(args []string) bool {
+	for _, arg := range args {
+		if arg == "-v" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func logsForTest(test, pkg string, all []LogLine) []LogLine {
+	var forTest []LogLine
+
+	for _, l := range all {
+		if l.Package == pkg && l.Test == test {
+			forTest = append(forTest, l)
+		}
+	}
+
+	return forTest
+}
+
+func topSlowest(max int, all []LogLine) []LogLine {
+	var top []LogLine
+
+	for _, l := range all {
+		if l.Test != "" && l.Elapsed > 0 {
+			top = append(top, l)
+		}
+	}
+
+	sort.Slice(top, func(i, j int) bool { return top[i].Elapsed > top[j].Elapsed })
+
+	if len(top) <= max {
+		return top
+	}
+	return top[0:max]
+}

examples/custom-tests/test/profile_structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no go binary'
+    path: '/usr/bin/go'
+    shouldExist: false

examples/custom-tests/test/structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no local go binary'
+    path: /usr/local/bin/go'
+    shouldExist: false