-
Notifications
You must be signed in to change notification settings - Fork 129
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add support to validate via Kong API #502
Changes from 7 commits
231e0e7
b91c844
ef450c4
4136209
422f066
bc31676
e25214e
a4c603e
9c3ec69
94ed516
ddeb657
7523bfd
a6b0a11
34fb225
3e13393
6319328
536ba04
809e7d3
9c68a3b
69135ea
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -1,29 +1,37 @@ | ||||||
package cmd | ||||||
|
||||||
import ( | ||||||
"context" | ||||||
"fmt" | ||||||
"reflect" | ||||||
"sync" | ||||||
|
||||||
"github.com/kong/deck/file" | ||||||
"github.com/kong/deck/state" | ||||||
"github.com/kong/deck/utils" | ||||||
"github.com/kong/go-kong/kong" | ||||||
"github.com/spf13/cobra" | ||||||
) | ||||||
|
||||||
var ( | ||||||
validateCmdKongStateFile []string | ||||||
validateCmdRBACResourcesOnly bool | ||||||
validateOnline bool | ||||||
) | ||||||
|
||||||
var maxConcurrency = 100 | ||||||
|
||||||
// validateCmd represents the diff command | ||||||
var validateCmd = &cobra.Command{ | ||||||
Use: "validate", | ||||||
Short: "Validate the state file", | ||||||
Long: `The validate command reads the state file and ensures validity. | ||||||
|
||||||
It reads all the specified state files and reports YAML/JSON | ||||||
parsing issues. It also checks for foreign relationships | ||||||
and alerts if there are broken relationships, or missing links present. | ||||||
|
||||||
No communication takes places between decK and Kong during the execution of | ||||||
this command. | ||||||
this command unless --online flag is used. | ||||||
`, | ||||||
Args: validateNoArgs, | ||||||
RunE: func(cmd *cobra.Command, args []string) error { | ||||||
|
@@ -50,11 +58,18 @@ this command. | |||||
return err | ||||||
} | ||||||
// this catches foreign relation errors | ||||||
_, err = state.Get(rawState) | ||||||
ks, err := state.Get(rawState) | ||||||
if err != nil { | ||||||
return err | ||||||
} | ||||||
|
||||||
if validateOnline { | ||||||
if errs := validateWithKong(cmd, ks); errs != nil { | ||||||
for _, e := range errs { | ||||||
fmt.Println(e) | ||||||
hbagdi marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
} | ||||||
} | ||||||
} | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. My bad, we don't print errors, we return the error from this command and cobra prints and exists appropriately. |
||||||
return nil | ||||||
}, | ||||||
PreRunE: func(cmd *cobra.Command, args []string) error { | ||||||
|
@@ -66,6 +81,128 @@ this command. | |||||
}, | ||||||
} | ||||||
|
||||||
func validateEntities(ctx context.Context, obj interface{}, kongClient *kong.Client, entityType string) []error { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Most of the functions that have been added in this PR should belong to either the util package or a new |
||||||
entities := callGetAll(obj) | ||||||
errors := []error{} | ||||||
|
||||||
// create a buffer of channels. Creation of new coroutines | ||||||
// are allowed only if the buffer is not full. | ||||||
chanBuff := make(chan struct{}, maxConcurrency) | ||||||
|
||||||
var wg sync.WaitGroup | ||||||
wg.Add(entities.Len()) | ||||||
hbagdi marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
// each coroutine will append on a slice of errors. | ||||||
// since slices are not thread-safe, let's add a mutex | ||||||
// to handle access to the slice. | ||||||
mu := &sync.Mutex{} | ||||||
for i := 0; i < entities.Len(); i++ { | ||||||
// reserve a slot | ||||||
chanBuff <- struct{}{} | ||||||
go func(i int) { | ||||||
defer wg.Done() | ||||||
// release a slot when completed | ||||||
defer func() { <-chanBuff }() | ||||||
_, err := validateEntity(ctx, kongClient, entityType, entities.Index(i).Interface()) | ||||||
if err != nil { | ||||||
mu.Lock() | ||||||
errors = append(errors, err) | ||||||
mu.Unlock() | ||||||
} | ||||||
}(i) | ||||||
} | ||||||
wg.Wait() | ||||||
return errors | ||||||
} | ||||||
|
||||||
func validateWithKong(cmd *cobra.Command, ks *state.KongState) []error { | ||||||
ctx := cmd.Context() | ||||||
kongClient, err := utils.GetKongClient(rootConfig) | ||||||
allErr := []error{} | ||||||
if err != nil { | ||||||
return []error{err} | ||||||
} | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add a call to |
||||||
if err := validateEntities(ctx, ks.Services, kongClient, "services"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.ACLGroups, kongClient, "acls"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.BasicAuths, kongClient, "basicauth_credentials"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.CACertificates, kongClient, "ca_certificates"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Certificates, kongClient, "certificates"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Consumers, kongClient, "consumers"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Documents, kongClient, "documents"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.HMACAuths, kongClient, "hmacauth_credentials"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.JWTAuths, kongClient, "jwt_secrets"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.KeyAuths, kongClient, "keyauth_credentials"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Oauth2Creds, kongClient, "oauth2_credentials"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Plugins, kongClient, "plugins"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Routes, kongClient, "routes"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.SNIs, kongClient, "snis"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Targets, kongClient, "targets"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
if err := validateEntities(ctx, ks.Upstreams, kongClient, "upstreams"); err != nil { | ||||||
allErr = append(allErr, err...) | ||||||
} | ||||||
return allErr | ||||||
} | ||||||
|
||||||
func callGetAll(obj interface{}) reflect.Value { | ||||||
// call GetAll method on entity | ||||||
method := reflect.ValueOf(obj).MethodByName("GetAll") | ||||||
entities := method.Call([]reflect.Value{})[0].Interface() | ||||||
return reflect.ValueOf(entities) | ||||||
} | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A further improvement to this function would be to add error as the second return value and check that in ensureGetAllMethod. An error should be returned if there is an error in looking up the method or calling the method (if any). This is not a blocker for merging. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah I totally agree an obscure fail will not help users. I implemented this (example by forcing a fail):
Let me know what you think! |
||||||
|
||||||
// ensureGetAllMethod ensures at init time that `GetAll()` method exists on the relevant structs. | ||||||
// If the method doesn't exist, the code will panic. This increases the likelihood of catching such an | ||||||
// error during manual testing. | ||||||
func ensureGetAllMethods() { | ||||||
// let's make sure ASAP that all resources have the expected GetAll method | ||||||
dummyEmptyState, _ := state.NewKongState() | ||||||
callGetAll(dummyEmptyState.Services) | ||||||
callGetAll(dummyEmptyState.ACLGroups) | ||||||
callGetAll(dummyEmptyState.BasicAuths) | ||||||
callGetAll(dummyEmptyState.CACertificates) | ||||||
callGetAll(dummyEmptyState.Certificates) | ||||||
callGetAll(dummyEmptyState.Consumers) | ||||||
callGetAll(dummyEmptyState.Documents) | ||||||
callGetAll(dummyEmptyState.HMACAuths) | ||||||
callGetAll(dummyEmptyState.JWTAuths) | ||||||
callGetAll(dummyEmptyState.KeyAuths) | ||||||
callGetAll(dummyEmptyState.Oauth2Creds) | ||||||
callGetAll(dummyEmptyState.Plugins) | ||||||
callGetAll(dummyEmptyState.Routes) | ||||||
callGetAll(dummyEmptyState.SNIs) | ||||||
callGetAll(dummyEmptyState.Targets) | ||||||
callGetAll(dummyEmptyState.Upstreams) | ||||||
hbagdi marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
} | ||||||
|
||||||
func init() { | ||||||
rootCmd.AddCommand(validateCmd) | ||||||
validateCmd.Flags().BoolVar(&validateCmdRBACResourcesOnly, "rbac-resources-only", | ||||||
|
@@ -74,4 +211,7 @@ func init() { | |||||
"state", "s", []string{"kong.yaml"}, "file(s) containing Kong's configuration.\n"+ | ||||||
"This flag can be specified multiple times for multiple files.\n"+ | ||||||
"Use '-' to read from stdin.") | ||||||
validateCmd.Flags().BoolVar(&validateOnline, "online", | ||||||
false, "perform schema validation against Kong API.") | ||||||
GGabriele marked this conversation as resolved.
Show resolved
Hide resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
ensureGetAllMethods() | ||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@hbagdi what value you think would be most appropriate here? For now I pseudo-randomly set it to
100
, which is slightly lower than the nginx default of128
.Another option would be to have a default value but make it configurable with a CLI flag.
Thoughts?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's make it configurable from the get-go. Add a
--parallelism
flag to the validate command.