feat: add support to validate via Kong API

cmd/common.go

@@ -26,6 +26,20 @@ var (
 	assumeYes  bool
 )
 
+func validateEntity(ctx context.Context, kongClient *kong.Client, entityType string, entity interface{}) (bool, error) {
+	errWrap := "validate entity '%s': %s"
+	endpoint := fmt.Sprintf("/schemas/%s/validate", entityType)
+	req, err := kongClient.NewRequest(http.MethodPost, endpoint, nil, entity)
+	if err != nil {
+		return false, fmt.Errorf(errWrap, entityType, err)
+	}
+	resp, err := kongClient.Do(ctx, req, nil)
+	if err != nil {
+		return false, fmt.Errorf(errWrap, entityType, err)
+	}
+	return resp.StatusCode == http.StatusOK, nil
+}
+
 // workspaceExists checks if workspace exists in Kong.
 func workspaceExists(ctx context.Context, config utils.KongClientConfig, workspaceName string) (bool, error) {
 	rootConfig := config.ForWorkspace("")
@@ -44,13 +58,22 @@ func workspaceExists(ctx context.Context, config utils.KongClientConfig, workspa
 		return false, err
 	}
 
-	exists, err := rootClient.Workspaces.ExistsByName(ctx, &workspaceName)
+	exists, err := rootClient.Workspaces.Exists(ctx, &workspaceName)
 	if err != nil {
 		return false, fmt.Errorf("checking if workspace exists: %w", err)
 	}
 	return exists, nil
 }
 
+func getWorkspaceName(workspace string, targetContent *file.Content) string {
+	if workspace != targetContent.Workspace && workspace != "" {
+		cprint.DeletePrintf("Warning: Workspace '%v' specified via --workspace flag is "+
+			"different from workspace '%v' found in state file(s).\n", workspace, targetContent.Workspace)
+		return workspace
+	}
+	return targetContent.Workspace
+}
+
 func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	delay int, workspace string) error {
 
@@ -65,16 +88,9 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return err
 	}
 
-	var wsConfig utils.KongClientConfig
-	var workspaceName string
 	// prepare to read the current state from Kong
-	if workspace != targetContent.Workspace && workspace != "" {
-		cprint.DeletePrintf("Warning: Workspace '%v' specified via --workspace flag is "+
-			"different from workspace '%v' found in state file(s).\n", workspace, targetContent.Workspace)
-		workspaceName = workspace
-	} else {
-		workspaceName = targetContent.Workspace
-	}
+	var wsConfig utils.KongClientConfig
+	workspaceName := getWorkspaceName(workspace, targetContent)
 	wsConfig = rootConfig.ForWorkspace(workspaceName)
 
 	// load Kong version after workspace

cmd/validate.go

@@ -1,29 +1,39 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"reflect"
+	"sync"
 
 	"github.com/kong/deck/file"
 	"github.com/kong/deck/state"
+	"github.com/kong/deck/utils"
+	"github.com/kong/go-kong/kong"
 	"github.com/spf13/cobra"
 )
 
 var (
 	validateCmdKongStateFile     []string
 	validateCmdRBACResourcesOnly bool
+	validateOnline               bool
+	validateWorkspace            string
 )
 
+var maxConcurrency = 100
+
 // validateCmd represents the diff command
 var validateCmd = &cobra.Command{
 	Use:   "validate",
 	Short: "Validate the state file",
 	Long: `The validate command reads the state file and ensures validity.
-
 It reads all the specified state files and reports YAML/JSON
 parsing issues. It also checks for foreign relationships
 and alerts if there are broken relationships, or missing links present.
+
 No communication takes places between decK and Kong during the execution of
-this command.
+this command unless --online flag is used.
 `,
 	Args: validateNoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -50,11 +60,19 @@ this command.
 			return err
 		}
 		// this catches foreign relation errors
-		_, err = state.Get(rawState)
+		ks, err := state.Get(rawState)
 		if err != nil {
 			return err
 		}
 
+		if validateOnline {
+			if errs := validateWithKong(cmd, ks, targetContent); errs != nil {
+				for _, e := range errs {
+					fmt.Println(e)
+				}
+				os.Exit(1)
+			}
+		}
 		return nil
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
@@ -66,6 +84,159 @@ this command.
 	},
 }
 
+func validateEntities(ctx context.Context, obj interface{}, kongClient *kong.Client, entityType string) []error {
+	entities := callGetAll(obj)
+	errors := []error{}
+
+	// create a buffer of channels. Creation of new coroutines
+	// are allowed only if the buffer is not full.
+	chanBuff := make(chan struct{}, maxConcurrency)
+
+	var wg sync.WaitGroup
+	wg.Add(entities.Len())
+	// each coroutine will append on a slice of errors.
+	// since slices are not thread-safe, let's add a mutex
+	// to handle access to the slice.
+	mu := &sync.Mutex{}
+	for i := 0; i < entities.Len(); i++ {
+		// reserve a slot
+		chanBuff <- struct{}{}
+		go func(i int) {
+			defer wg.Done()
+			// release a slot when completed
+			defer func() { <-chanBuff }()
+			_, err := validateEntity(ctx, kongClient, entityType, entities.Index(i).Interface())
+			if err != nil {
+				mu.Lock()
+				errors = append(errors, err)
+				mu.Unlock()
+			}
+		}(i)
+	}
+	wg.Wait()
+	return errors
+}
+
+func validateWithKong(cmd *cobra.Command, ks *state.KongState, targetContent *file.Content) []error {
+	ctx := cmd.Context()
+	// make sure we are able to connect to Kong
+	_, err := fetchKongVersion(ctx, rootConfig)
+	if err != nil {
+		return []error{fmt.Errorf("couldn't fetch Kong version: %w", err)}
+	}
+
+	// check if workspace exists
+	workspaceName := getWorkspaceName(validateWorkspace, targetContent)
+	workspaceExists, err := workspaceExists(ctx, rootConfig, workspaceName)
+	if err != nil {
+		return []error{err}
+	}
+	if !workspaceExists {
+		return []error{fmt.Errorf("workspace doesn't exist: %s", workspaceName)}
+	}
+
+	wsConfig := rootConfig.ForWorkspace(workspaceName)
+	kongClient, err := utils.GetKongClient(wsConfig)
+	allErr := []error{}
+	if err != nil {
+		return []error{err}
+	}
+
+	// validate RBAC resources first.
+	if err := validateEntities(ctx, ks.RBACEndpointPermissions, kongClient, "rbac-endpointpermission"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.RBACRoles, kongClient, "rbac-role"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if validateCmdRBACResourcesOnly {
+		return allErr
+	}
+
+	if err := validateEntities(ctx, ks.Services, kongClient, "services"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.ACLGroups, kongClient, "acls"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.BasicAuths, kongClient, "basicauth_credentials"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.CACertificates, kongClient, "ca_certificates"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Certificates, kongClient, "certificates"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Consumers, kongClient, "consumers"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Documents, kongClient, "documents"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.HMACAuths, kongClient, "hmacauth_credentials"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.JWTAuths, kongClient, "jwt_secrets"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.KeyAuths, kongClient, "keyauth_credentials"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Oauth2Creds, kongClient, "oauth2_credentials"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Plugins, kongClient, "plugins"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Routes, kongClient, "routes"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.SNIs, kongClient, "snis"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Targets, kongClient, "targets"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	if err := validateEntities(ctx, ks.Upstreams, kongClient, "upstreams"); err != nil {
+		allErr = append(allErr, err...)
+	}
+	return allErr
+}
+
+func callGetAll(obj interface{}) reflect.Value {
+	// call GetAll method on entity
+	method := reflect.ValueOf(obj).MethodByName("GetAll")
+	entities := method.Call([]reflect.Value{})[0].Interface()
+	return reflect.ValueOf(entities)
+}
+
+// ensureGetAllMethod ensures at init time that `GetAll()` method exists on the relevant structs.
+// If the method doesn't exist, the code will panic. This increases the likelihood of catching such an
+// error during manual testing.
+func ensureGetAllMethods() {
+	// let's make sure ASAP that all resources have the expected GetAll method
+	dummyEmptyState, _ := state.NewKongState()
+	callGetAll(dummyEmptyState.Services)
+	callGetAll(dummyEmptyState.ACLGroups)
+	callGetAll(dummyEmptyState.BasicAuths)
+	callGetAll(dummyEmptyState.CACertificates)
+	callGetAll(dummyEmptyState.Certificates)
+	callGetAll(dummyEmptyState.Consumers)
+	callGetAll(dummyEmptyState.Documents)
+	callGetAll(dummyEmptyState.HMACAuths)
+	callGetAll(dummyEmptyState.JWTAuths)
+	callGetAll(dummyEmptyState.KeyAuths)
+	callGetAll(dummyEmptyState.Oauth2Creds)
+	callGetAll(dummyEmptyState.Plugins)
+	callGetAll(dummyEmptyState.Routes)
+	callGetAll(dummyEmptyState.SNIs)
+	callGetAll(dummyEmptyState.Targets)
+	callGetAll(dummyEmptyState.Upstreams)
+	callGetAll(dummyEmptyState.RBACEndpointPermissions)
+	callGetAll(dummyEmptyState.RBACRoles)
+}
+
 func init() {
 	rootCmd.AddCommand(validateCmd)
 	validateCmd.Flags().BoolVar(&validateCmdRBACResourcesOnly, "rbac-resources-only",
@@ -74,4 +245,11 @@ func init() {
 		"state", "s", []string{"kong.yaml"}, "file(s) containing Kong's configuration.\n"+
 			"This flag can be specified multiple times for multiple files.\n"+
 			"Use '-' to read from stdin.")
+	validateCmd.Flags().BoolVar(&validateOnline, "online",
+		false, "perform schema validation against Kong API.")
+	validateCmd.Flags().StringVarP(&validateWorkspace, "workspace", "w",
+		"", "validate configuration of a specific workspace "+
+			"(Kong Enterprise only).\n"+
+			"This takes precedence over _workspace fields in state files.")
+	ensureGetAllMethods()
 }

main.go

@@ -18,6 +18,7 @@ func registerSignalHandler() context.Context {
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
 	go func() {
+		defer signal.Stop(sigs)
 		sig := <-sigs
 		fmt.Println("received", sig, ", terminating...")
 		cancel()