diff --git a/cmd/gator/test/test.go b/cmd/gator/test/test.go index d9057ffd559..b274d363f44 100644 --- a/cmd/gator/test/test.go +++ b/cmd/gator/test/test.go @@ -17,17 +17,17 @@ const ( examples = ` # Run all tests in label-tests.yaml gator test label-tests.yaml - # Run all suites whose names contain "forbid-labels". + # Run all tests whose names contain "forbid-labels". gator test tests/... --run forbid-labels// - # Run all tests whose names contain "nginx-deployment". + # Run all cases whose names contain "nginx-deployment". gator test tests/... --run //nginx-deployment - # Run all tests whose names exactly match "nginx-deployment". + # Run all cases whose names exactly match "nginx-deployment". gator test tests/... --run '//^nginx-deployment$' - # Run all tests that are either named "forbid-labels" or are - # in suites named "forbid-labels". + # Run all cases that are either named "forbid-labels" or are + # in tests named "forbid-labels". gator test tests/... --run '^forbid-labels$'` ) @@ -87,15 +87,16 @@ func runE(cmd *cobra.Command, args []string) error { func runSuites(ctx context.Context, fileSystem fs.FS, suites []gktest.Suite, filter gktest.Filter) error { isFailure := false + + runner := gktest.Runner{ + FS: fileSystem, + NewClient: gktest.NewOPAClient, + } + for i := range suites { s := &suites[i] - c, err := gktest.NewOPAClient() - if err != nil { - return err - } - - suiteResult := s.Run(ctx, c, fileSystem, filter) + suiteResult := runner.Run(ctx, filter, s) for _, testResult := range suiteResult.TestResults { if testResult.Error != nil { isFailure = true diff --git a/pkg/gktest/errors.go b/pkg/gktest/errors.go new file mode 100644 index 00000000000..dd0545e017f --- /dev/null +++ b/pkg/gktest/errors.go @@ -0,0 +1,21 @@ +package gktest + +import "errors" + +var ( + // ErrNotATemplate indicates the user-indicated file does not contain a + // ConstraintTemplate. + ErrNotATemplate = errors.New("not a ConstraintTemplate") + // ErrNotAConstraint indicates the user-indicated file does not contain a + // Constraint. + ErrNotAConstraint = errors.New("not a Constraint") + // ErrAddingTemplate indicates a problem instantiating a Suite's ConstraintTemplate. + ErrAddingTemplate = errors.New("adding template") + // ErrAddingConstraint indicates a problem instantiating a Suite's Constraint. + ErrAddingConstraint = errors.New("adding constraint") + // ErrInvalidSuite indicates a Suite does not define the required fields. + ErrInvalidSuite = errors.New("invalid Suite") + // ErrCreatingClient indicates an error instantiating the Client which compiles + // Constraints and runs validation. + ErrCreatingClient = errors.New("creating client") +) diff --git a/pkg/gktest/opa.go b/pkg/gktest/opa.go index 89ec6f3feb4..32c39088d54 100644 --- a/pkg/gktest/opa.go +++ b/pkg/gktest/opa.go @@ -6,7 +6,7 @@ import ( "github.com/open-policy-agent/gatekeeper/pkg/target" ) -func NewOPAClient() (*opaclient.Client, error) { +func NewOPAClient() (Client, error) { driver := local.New(local.Tracing(false)) backend, err := opaclient.NewBackend(opaclient.Driver(driver)) if err != nil { diff --git a/pkg/gktest/read_constraints.go b/pkg/gktest/read_constraints.go new file mode 100644 index 00000000000..a6c73f82e33 --- /dev/null +++ b/pkg/gktest/read_constraints.go @@ -0,0 +1,96 @@ +package gktest + +import ( + "encoding/json" + "fmt" + "io/fs" + + templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1" + "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" + "github.com/open-policy-agent/gatekeeper/apis" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// scheme stores the k8s resource types we can instantiate as Templates. +var scheme = runtime.NewScheme() + +func init() { + _ = apis.AddToScheme(scheme) +} + +// readTemplate reads the contents of the path and returns the +// ConstraintTemplate it defines. Returns an error if the file does not define +// a ConstraintTemplate. +func readTemplate(f fs.FS, path string) (*templates.ConstraintTemplate, error) { + bytes, err := fs.ReadFile(f, path) + if err != nil { + return nil, fmt.Errorf("reading ConstraintTemplate from %q: %w", path, err) + } + + u := unstructured.Unstructured{ + Object: make(map[string]interface{}), + } + err = yaml.Unmarshal(bytes, u.Object) + if err != nil { + return nil, fmt.Errorf("%w: parsing ConstraintTemplate YAML from %q: %v", ErrAddingTemplate, path, err) + } + + gvk := u.GroupVersionKind() + if gvk.Group != templatesv1.SchemeGroupVersion.Group || gvk.Kind != "ConstraintTemplate" { + return nil, fmt.Errorf("%w: %q", ErrNotATemplate, path) + } + + t, err := scheme.New(gvk) + if err != nil { + // The type isn't registered in the scheme. + return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err) + } + + // YAML parsing doesn't properly handle ObjectMeta, so we must + // marshal/unmashal through JSON. + jsonBytes, err := u.MarshalJSON() + if err != nil { + // Indicates a bug in unstructured.MarshalJSON(). Any Unstructured + // unmarshalled from YAML should be marshallable to JSON. + return nil, fmt.Errorf("calling unstructured.MarshalJSON(): %w", err) + } + err = json.Unmarshal(jsonBytes, t) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err) + } + + template := &templates.ConstraintTemplate{} + err = scheme.Convert(t, template, nil) + if err != nil { + // This shouldn't happen unless there's a bug in the conversion functions. + // Most likely it means the conversion functions weren't generated. + return nil, err + } + + return template, nil +} + +func readConstraint(f fs.FS, path string) (*unstructured.Unstructured, error) { + bytes, err := fs.ReadFile(f, path) + if err != nil { + return nil, fmt.Errorf("reading Constraint from %q: %w", path, err) + } + + c := &unstructured.Unstructured{ + Object: make(map[string]interface{}), + } + + err = yaml.Unmarshal(bytes, c.Object) + if err != nil { + return nil, fmt.Errorf("%w: parsing Constraint from %q: %v", ErrAddingConstraint, path, err) + } + + gvk := c.GroupVersionKind() + if gvk.Group != "constraints.gatekeeper.sh" { + return nil, ErrNotAConstraint + } + + return c, nil +} diff --git a/pkg/gktest/runner.go b/pkg/gktest/runner.go new file mode 100644 index 00000000000..6cec19564e4 --- /dev/null +++ b/pkg/gktest/runner.go @@ -0,0 +1,80 @@ +package gktest + +import ( + "context" + "fmt" + "io/fs" +) + +// Runner defines logic independent of how tests are run and the results are +// printed. +type Runner struct { + // FS is the filesystem the Runner interacts with to read Suites and objects. + FS fs.FS + + // NewClient instantiates a Client for compiling Templates/Constraints, and + // validating objects against them. + NewClient func() (Client, error) + + // TODO: Add Printer. +} + +// Run executes all Tests in the Suite and returns the results. +func (r *Runner) Run(ctx context.Context, filter Filter, s *Suite) SuiteResult { + result := SuiteResult{ + TestResults: make([]TestResult, len(s.Tests)), + } + for i, t := range s.Tests { + if filter.MatchesTest(t) { + result.TestResults[i] = r.runTest(ctx, filter, t) + } + } + return result +} + +// runTest executes every Case in the Test. Returns the results for every Case. +func (r *Runner) runTest(ctx context.Context, filter Filter, t Test) TestResult { + client, err := r.NewClient() + if err != nil { + return TestResult{Error: fmt.Errorf("%w: %v", ErrCreatingClient, err)} + } + + if t.Template == "" { + return TestResult{Error: fmt.Errorf("%w: missing template", ErrInvalidSuite)} + } + template, err := readTemplate(r.FS, t.Template) + if err != nil { + return TestResult{Error: err} + } + _, err = client.AddTemplate(ctx, template) + if err != nil { + return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingTemplate, err)} + } + + if t.Constraint == "" { + return TestResult{Error: fmt.Errorf("%w: missing constraint", ErrInvalidSuite)} + } + cObj, err := readConstraint(r.FS, t.Constraint) + if err != nil { + return TestResult{Error: err} + } + _, err = client.AddConstraint(ctx, cObj) + if err != nil { + return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingConstraint, err)} + } + + results := make([]CaseResult, len(t.Cases)) + for i, c := range t.Cases { + if !filter.MatchesCase(c) { + continue + } + + results[i] = r.runCase(ctx, client, c) + } + return TestResult{CaseResults: results} +} + +// RunCase executes a Case and returns the result of the run. +func (r *Runner) runCase(ctx context.Context, client Client, c Case) CaseResult { + return CaseResult{} +} diff --git a/pkg/gktest/suite_test.go b/pkg/gktest/runner_test.go similarity index 90% rename from pkg/gktest/suite_test.go rename to pkg/gktest/runner_test.go index b73789deef9..6b44809e59e 100644 --- a/pkg/gktest/suite_test.go +++ b/pkg/gktest/runner_test.go @@ -2,6 +2,7 @@ package gktest import ( "context" + "errors" "io/fs" "testing" "testing/fstest" @@ -119,7 +120,7 @@ metadata: ` ) -func TestSuite_Run(t *testing.T) { +func TestRunner_Run(t *testing.T) { testCases := []struct { name string suite Suite @@ -266,6 +267,7 @@ func TestSuite_Run(t *testing.T) { Tests: []Test{{ Template: "template.yaml", Constraint: "constraint.yaml", + Cases: []Case{{}}, }}, }, f: fstest.MapFS{ @@ -277,7 +279,9 @@ func TestSuite_Run(t *testing.T) { }, }, want: SuiteResult{ - TestResults: []TestResult{{}}, + TestResults: []TestResult{{ + CaseResults: []CaseResult{{}}, + }}, }, }, { @@ -370,13 +374,13 @@ func TestSuite_Run(t *testing.T) { for _, tc := range testCases { ctx := context.Background() - c, err := NewOPAClient() - if err != nil { - t.Fatal(err) + runner := Runner{ + FS: tc.f, + NewClient: NewOPAClient, } t.Run(tc.name, func(t *testing.T) { - got := tc.suite.Run(ctx, c, tc.f, Filter{}) + got := runner.Run(ctx, Filter{}, &tc.suite) if diff := cmp.Diff(tc.want, got, cmpopts.EquateErrors(), cmpopts.EquateEmpty()); diff != "" { t.Errorf(diff) @@ -384,3 +388,27 @@ func TestSuite_Run(t *testing.T) { }) } } + +func TestRunner_Run_ClientError(t *testing.T) { + want := SuiteResult{ + TestResults: []TestResult{{Error: ErrCreatingClient}}, + } + + runner := Runner{ + FS: fstest.MapFS{}, + NewClient: func() (Client, error) { + return nil, errors.New("error") + }, + } + + ctx := context.Background() + + suite := &Suite{ + Tests: []Test{{}}, + } + got := runner.Run(ctx, Filter{}, suite) + + if diff := cmp.Diff(want, got, cmpopts.EquateErrors(), cmpopts.EquateEmpty()); diff != "" { + t.Error(diff) + } +} diff --git a/pkg/gktest/suite.go b/pkg/gktest/suite.go index 08f8d3965ca..cc76d80f5a7 100644 --- a/pkg/gktest/suite.go +++ b/pkg/gktest/suite.go @@ -1,28 +1,9 @@ package gktest import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - - "github.com/open-policy-agent/frameworks/constraint/pkg/apis" - templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1" - "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" - "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" ) -// scheme stores the k8s resource types we can instantiate as Templates. -var scheme = runtime.NewScheme() - -func init() { - _ = apis.AddToScheme(scheme) -} - // Suite defines a set of Constraint tests. type Suite struct { metav1.ObjectMeta @@ -32,19 +13,6 @@ type Suite struct { Tests []Test } -// Run executes all Tests in the Suite and returns the results. -func (s *Suite) Run(ctx context.Context, client Client, f fs.FS, filter Filter) SuiteResult { - result := SuiteResult{ - TestResults: make([]TestResult, len(s.Tests)), - } - for i, c := range s.Tests { - if filter.MatchesTest(c) { - result.TestResults[i] = c.run(ctx, client, f, filter) - } - } - return result -} - // Test defines a Template&Constraint pair to instantiate, and Cases to // run on the instantiated Constraint. type Test struct { @@ -60,137 +28,5 @@ type Test struct { Cases []Case } -var ( - // ErrNotATemplate indicates the user-indicated file does not contain a - // ConstraintTemplate. - ErrNotATemplate = errors.New("not a ConstraintTemplate") - // ErrNotAConstraint indicates the user-indicated file does not contain a - // Constraint. - ErrNotAConstraint = errors.New("not a Constraint") - // ErrAddingTemplate indicates a problem instantiating a Suite's ConstraintTemplate. - ErrAddingTemplate = errors.New("adding template") - // ErrAddingConstraint indicates a problem instantiating a Suite's Constraint. - ErrAddingConstraint = errors.New("adding constraint") - // ErrInvalidSuite indicates a Suite does not define the required fields. - ErrInvalidSuite = errors.New("invalid Suite") -) - -// readTemplate reads the contents of the path and returns the -// ConstraintTemplate it defines. Returns an error if the file does not define -// a ConstraintTemplate. -func readTemplate(f fs.FS, path string) (*templates.ConstraintTemplate, error) { - bytes, err := fs.ReadFile(f, path) - if err != nil { - return nil, fmt.Errorf("reading ConstraintTemplate from %q: %w", path, err) - } - - u := unstructured.Unstructured{ - Object: make(map[string]interface{}), - } - err = yaml.Unmarshal(bytes, u.Object) - if err != nil { - return nil, fmt.Errorf("%w: parsing ConstraintTemplate YAML from %q: %v", ErrAddingTemplate, path, err) - } - - gvk := u.GroupVersionKind() - if gvk.Group != templatesv1.SchemeGroupVersion.Group || gvk.Kind != "ConstraintTemplate" { - return nil, fmt.Errorf("%w: %q", ErrNotATemplate, path) - } - - t, err := scheme.New(gvk) - if err != nil { - // The type isn't registered in the scheme. - return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err) - } - - // YAML parsing doesn't properly handle ObjectMeta, so we must - // marshal/unmashal through JSON. - jsonBytes, err := u.MarshalJSON() - if err != nil { - // Indicates a bug in unstructured.MarshalJSON(). Any Unstructured - // unmarshalled from YAML should be marshallable to JSON. - return nil, fmt.Errorf("calling unstructured.MarshalJSON(): %w", err) - } - err = json.Unmarshal(jsonBytes, t) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrAddingTemplate, err) - } - - template := &templates.ConstraintTemplate{} - err = scheme.Convert(t, template, nil) - if err != nil { - // This shouldn't happen unless there's a bug in the conversion functions. - // Most likely it means the conversion functions weren't generated. - return nil, err - } - - return template, nil -} - -func readConstraint(f fs.FS, path string) (*unstructured.Unstructured, error) { - bytes, err := fs.ReadFile(f, path) - if err != nil { - return nil, fmt.Errorf("reading Constraint from %q: %w", path, err) - } - - c := &unstructured.Unstructured{ - Object: make(map[string]interface{}), - } - - err = yaml.Unmarshal(bytes, c.Object) - if err != nil { - return nil, fmt.Errorf("%w: parsing Constraint from %q: %v", ErrAddingConstraint, path, err) - } - - gvk := c.GroupVersionKind() - if gvk.Group != "constraints.gatekeeper.sh" { - return nil, ErrNotAConstraint - } - - return c, nil -} - -// run executes every Case in the Test. Returns the results for every Case. -func (t Test) run(ctx context.Context, client Client, f fs.FS, filter Filter) TestResult { - if t.Template == "" { - return TestResult{Error: fmt.Errorf("%w: missing template", ErrInvalidSuite)} - } - template, err := readTemplate(f, t.Template) - if err != nil { - return TestResult{Error: err} - } - _, err = client.AddTemplate(ctx, template) - if err != nil { - return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingTemplate, err)} - } - - if t.Constraint == "" { - return TestResult{Error: fmt.Errorf("%w: missing constraint", ErrInvalidSuite)} - } - cObj, err := readConstraint(f, t.Constraint) - if err != nil { - return TestResult{Error: err} - } - _, err = client.AddConstraint(ctx, cObj) - if err != nil { - return TestResult{Error: fmt.Errorf("%w: %v", ErrAddingConstraint, err)} - } - - results := make([]CaseResult, len(t.Cases)) - for i, tc := range t.Cases { - if !filter.MatchesCase(tc) { - continue - } - - results[i] = tc.run(f, client) - } - return TestResult{CaseResults: results} -} - // Case runs Constraint against a YAML object. type Case struct{} - -// run executes the Case and returns the Result of the run. -func (c Case) run(f fs.FS, client Client) CaseResult { - return CaseResult{} -}