[gk-test] Refactor run code into Runner

cmd/gator/test/test.go

@@ -17,17 +17,17 @@ const (
 	examples = `  # Run all tests in label-tests.yaml
   gator test label-tests.yaml
 
-  # Run all suites whose names contain "forbid-labels".
+  # Run all tests whose names contain "forbid-labels".
   gator test tests/... --run forbid-labels//
 
-  # Run all tests whose names contain "nginx-deployment".
+  # Run all cases whose names contain "nginx-deployment".
   gator test tests/... --run //nginx-deployment
 
-  # Run all tests whose names exactly match "nginx-deployment".
+  # Run all cases whose names exactly match "nginx-deployment".
   gator test tests/... --run '//^nginx-deployment$'
 
-  # Run all tests that are either named "forbid-labels" or are
-  # in suites named "forbid-labels".
+  # Run all cases that are either named "forbid-labels" or are
+  # in tests named "forbid-labels".
   gator test tests/... --run '^forbid-labels$'`
 )
 
@@ -87,15 +87,16 @@ func runE(cmd *cobra.Command, args []string) error {
 
 func runSuites(ctx context.Context, fileSystem fs.FS, suites []gktest.Suite, filter gktest.Filter) error {
 	isFailure := false
+
+	runner := gktest.Runner{
+		FS:        fileSystem,
+		NewClient: gktest.NewOPAClient,
+	}
+
 	for i := range suites {
 		s := &suites[i]
 
-		c, err := gktest.NewOPAClient()
-		if err != nil {
-			return err
-		}
-
-		suiteResult := s.Run(ctx, c, fileSystem, filter)
+		suiteResult := runner.Run(ctx, filter, s)
 		for _, testResult := range suiteResult.TestResults {
 			if testResult.Error != nil {
 				isFailure = true

pkg/gktest/errors.go

@@ -0,0 +1,21 @@
+package gktest
+
+import "errors"
+
+var (
+	// ErrNotATemplate indicates the user-indicated file does not contain a
+	// ConstraintTemplate.
+	ErrNotATemplate = errors.New("not a ConstraintTemplate")
+	// ErrNotAConstraint indicates the user-indicated file does not contain a
+	// Constraint.
+	ErrNotAConstraint = errors.New("not a Constraint")
+	// ErrAddingTemplate indicates a problem instantiating a Suite's ConstraintTemplate.
+	ErrAddingTemplate = errors.New("adding template")
+	// ErrAddingConstraint indicates a problem instantiating a Suite's Constraint.
+	ErrAddingConstraint = errors.New("adding constraint")
+	// ErrInvalidSuite indicates a Suite does not define the required fields.
+	ErrInvalidSuite = errors.New("invalid Suite")
+	// ErrCreatingClient indicates an error instantiating the Client which compiles
+	// Constraints and runs validation.
+	ErrCreatingClient = errors.New("creating client")
+)

pkg/gktest/opa.go

@@ -6,7 +6,7 @@ import (
 	"github.com/open-policy-agent/gatekeeper/pkg/target"
 )
 
-func NewOPAClient() (*opaclient.Client, error) {
+func NewOPAClient() (Client, error) {
 	driver := local.New(local.Tracing(false))
 	backend, err := opaclient.NewBackend(opaclient.Driver(driver))
 	if err != nil {

pkg/gktest/read_constraints.go

@@ -0,0 +1,96 @@
+package gktest
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+
+	templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1"
+	"github.com/open-policy-agent/frameworks/constraint/pkg/core/templates"
+	"github.com/open-policy-agent/gatekeeper/apis"
+	"gopkg.in/yaml.v3"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// scheme stores the k8s resource types we can instantiate as Templates.
+var scheme = runtime.NewScheme()
+
+func init() {
+	_ = apis.AddToScheme(scheme)
+}
+
+// readTemplate reads the contents of the path and returns the
+// ConstraintTemplate it defines. Returns an error if the file does not define
+// a ConstraintTemplate.
+func readTemplate(f fs.FS, path string) (*templates.ConstraintTemplate, error) {
+	bytes, err := fs.ReadFile(f, path)
+	if err != nil {
+		return nil, fmt.Errorf("reading ConstraintTemplate from %q: %w", path, err)
+	}
+
+	u := unstructured.Unstructured{
+		Object: make(map[string]interface{}),
+	}
+	err = yaml.Unmarshal(bytes, u.Object)
+	if err != nil {
+		return nil, fmt.Errorf("%w: parsing ConstraintTemplate YAML from %q: %v", ErrAddingTemplate, path, err)
+	}
+
+	gvk := u.GroupVersionKind()
+	if gvk.Group != templatesv1.SchemeGroupVersion.Group || gvk.Kind != "ConstraintTemplate" {
+		return nil, fmt.Errorf("%w: %q", ErrNotATemplate, path)
+	}
+
+	t, err := scheme.New(gvk)
+	if err != nil {
+		// The type isn't registered in the scheme.
+		return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err)
+	}
+
+	// YAML parsing doesn't properly handle ObjectMeta, so we must
+	// marshal/unmashal through JSON.
+	jsonBytes, err := u.MarshalJSON()
+	if err != nil {
+		// Indicates a bug in unstructured.MarshalJSON(). Any Unstructured
+		// unmarshalled from YAML should be marshallable to JSON.
+		return nil, fmt.Errorf("calling unstructured.MarshalJSON(): %w", err)
+	}
+	err = json.Unmarshal(jsonBytes, t)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err)
+	}
+
+	template := &templates.ConstraintTemplate{}
+	err = scheme.Convert(t, template, nil)
+	if err != nil {
+		// This shouldn't happen unless there's a bug in the conversion functions.
+		// Most likely it means the conversion functions weren't generated.
+		return nil, err
+	}
+
+	return template, nil
+}
+
+func readConstraint(f fs.FS, path string) (*unstructured.Unstructured, error) {
+	bytes, err := fs.ReadFile(f, path)
+	if err != nil {
+		return nil, fmt.Errorf("reading Constraint from %q: %w", path, err)
+	}
+
+	c := &unstructured.Unstructured{
+		Object: make(map[string]interface{}),
+	}
+
+	err = yaml.Unmarshal(bytes, c.Object)
+	if err != nil {
+		return nil, fmt.Errorf("%w: parsing Constraint from %q: %v", ErrAddingConstraint, path, err)
+	}
+
+	gvk := c.GroupVersionKind()
+	if gvk.Group != "constraints.gatekeeper.sh" {
+		return nil, ErrNotAConstraint
+	}
+
+	return c, nil
+}

pkg/gktest/runner.go

@@ -0,0 +1,80 @@
+package gktest
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+)
+
+// Runner defines logic independent of how tests are run and the results are
+// printed.
+type Runner struct {
+	// FS is the filesystem the Runner interacts with to read Suites and objects.
+	FS fs.FS
+
+	// NewClient instantiates a Client for compiling Templates/Constraints, and
+	// validating objects against them.
+	NewClient func() (Client, error)
+
+	// TODO: Add Printer.
+}
+
+// Run executes all Tests in the Suite and returns the results.
+func (r *Runner) Run(ctx context.Context, filter Filter, s *Suite) SuiteResult {
+	result := SuiteResult{
+		TestResults: make([]TestResult, len(s.Tests)),
+	}
+	for i, t := range s.Tests {
+		if filter.MatchesTest(t) {
+			result.TestResults[i] = r.runTest(ctx, filter, t)
+		}
+	}
+	return result
+}
+
+// runTest executes every Case in the Test. Returns the results for every Case.
+func (r *Runner) runTest(ctx context.Context, filter Filter, t Test) TestResult {
+	client, err := r.NewClient()
+	if err != nil {
+		return TestResult{Error: fmt.Errorf("%w: %v", ErrCreatingClient, err)}
+	}
+
+	if t.Template == "" {
+		return TestResult{Error: fmt.Errorf("%w: missing template", ErrInvalidSuite)}
+	}
+	template, err := readTemplate(r.FS, t.Template)
+	if err != nil {
+		return TestResult{Error: err}
+	}
+	_, err = client.AddTemplate(ctx, template)
+	if err != nil {
+		return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingTemplate, err)}
+	}
+
+	if t.Constraint == "" {
+		return TestResult{Error: fmt.Errorf("%w: missing constraint", ErrInvalidSuite)}
+	}
+	cObj, err := readConstraint(r.FS, t.Constraint)
+	if err != nil {
+		return TestResult{Error: err}
+	}
+	_, err = client.AddConstraint(ctx, cObj)
+	if err != nil {
+		return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingConstraint, err)}
+	}
+
+	results := make([]CaseResult, len(t.Cases))
+	for i, c := range t.Cases {
+		if !filter.MatchesCase(c) {
+			continue
+		}
+
+		results[i] = r.runCase(ctx, client, c)
+	}
+	return TestResult{CaseResults: results}
+}
+
+// RunCase executes a Case and returns the result of the run.
+func (r *Runner) runCase(ctx context.Context, client Client, c Case) CaseResult {
+	return CaseResult{}
+}

pkg/gktest/suite_test.go → pkg/gktest/runner_test.go

@@ -2,6 +2,7 @@ package gktest
 
 import (
 	"context"
+	"errors"
 	"io/fs"
 	"testing"
 	"testing/fstest"
@@ -119,7 +120,7 @@ metadata:
 `
 )
 
-func TestSuite_Run(t *testing.T) {
+func TestRunner_Run(t *testing.T) {
 	testCases := []struct {
 		name  string
 		suite Suite
@@ -266,6 +267,7 @@ func TestSuite_Run(t *testing.T) {
 				Tests: []Test{{
 					Template:   "template.yaml",
 					Constraint: "constraint.yaml",
+					Cases:      []Case{{}},
 				}},
 			},
 			f: fstest.MapFS{
@@ -277,7 +279,9 @@ func TestSuite_Run(t *testing.T) {
 				},
 			},
 			want: SuiteResult{
-				TestResults: []TestResult{{}},
+				TestResults: []TestResult{{
+					CaseResults: []CaseResult{{}},
+				}},
 			},
 		},
 		{
@@ -370,17 +374,41 @@ func TestSuite_Run(t *testing.T) {
 	for _, tc := range testCases {
 		ctx := context.Background()
 
-		c, err := NewOPAClient()
-		if err != nil {
-			t.Fatal(err)
+		runner := Runner{
+			FS:        tc.f,
+			NewClient: NewOPAClient,
 		}
 
 		t.Run(tc.name, func(t *testing.T) {
-			got := tc.suite.Run(ctx, c, tc.f, Filter{})
+			got := runner.Run(ctx, Filter{}, &tc.suite)
 
 			if diff := cmp.Diff(tc.want, got, cmpopts.EquateErrors(), cmpopts.EquateEmpty()); diff != "" {
 				t.Errorf(diff)
 			}
 		})
 	}
 }
+
+func TestRunner_Run_ClientError(t *testing.T) {
+	want := SuiteResult{
+		TestResults: []TestResult{{Error: ErrCreatingClient}},
+	}
+
+	runner := Runner{
+		FS: fstest.MapFS{},
+		NewClient: func() (Client, error) {
+			return nil, errors.New("error")
+		},
+	}
+
+	ctx := context.Background()
+
+	suite := &Suite{
+		Tests: []Test{{}},
+	}
+	got := runner.Run(ctx, Filter{}, suite)
+
+	if diff := cmp.Diff(want, got, cmpopts.EquateErrors(), cmpopts.EquateEmpty()); diff != "" {
+		t.Error(diff)
+	}
+}