diff --git a/cmd/common.go b/cmd/common.go index 2f9966add..bf2fd19b1 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -81,7 +81,7 @@ func getWorkspaceName(workspaceFlag string, targetContent *file.Content) string } func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, - delay int, workspace string, + delay int, workspace string, noMaskValues bool, ) error { // read target file targetContent, err := file.GetContentFromFiles(filenames) @@ -225,7 +225,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, return err } - totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient) + totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient, noMaskValues) if err != nil { return err } @@ -273,7 +273,7 @@ func fetchCurrentState(ctx context.Context, client *kong.Client, dumpConfig dump } func performDiff(ctx context.Context, currentState, targetState *state.KongState, - dry bool, parallelism int, delay int, client *kong.Client, + dry bool, parallelism int, delay int, client *kong.Client, noMaskValues bool, ) (int, error) { s, err := diff.NewSyncer(diff.SyncerOpts{ CurrentState: currentState, @@ -285,7 +285,7 @@ func performDiff(ctx context.Context, currentState, targetState *state.KongState return 0, err } - stats, errs := s.Solve(ctx, parallelism, dry) + stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go index de5e6207c..0e7b04fdd 100644 --- a/cmd/common_konnect.go +++ b/cmd/common_konnect.go @@ -119,7 +119,7 @@ func getKongClientForKonnectMode(ctx context.Context) (*kong.Client, error) { }) } -func resetKonnectV2(ctx context.Context) error { +func resetKonnectV2(ctx context.Context, resetNoMaskDeckEnvVarsValue bool) error { client, err := getKongClientForKonnectMode(ctx) if err != nil { return err @@ -132,7 +132,7 @@ func resetKonnectV2(ctx context.Context) error { if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client, resetNoMaskDeckEnvVarsValue) if err != nil { return err } @@ -166,7 +166,7 @@ func dumpKonnectV2(ctx context.Context) error { } func syncKonnect(ctx context.Context, - filenames []string, dry bool, parallelism int, + filenames []string, dry bool, parallelism int, noMaskValues bool, ) error { httpClient := utils.HTTPClient() @@ -245,7 +245,7 @@ func syncKonnect(ctx context.Context, return err } - stats, errs := s.Solve(ctx, parallelism, dry) + stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/diff.go b/cmd/diff.go index d6055b68c..02d490b23 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -7,10 +7,11 @@ import ( ) var ( - diffCmdKongStateFile []string - diffCmdParallelism int - diffCmdNonZeroExitCode bool - diffWorkspace string + diffCmdKongStateFile []string + diffCmdParallelism int + diffCmdNoMaskDeckEnvVarsValue bool + diffCmdNonZeroExitCode bool + diffWorkspace string ) // newDiffCmd represents the diff command @@ -27,7 +28,7 @@ that will be created, updated, or deleted. Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), diffCmdKongStateFile, true, - diffCmdParallelism, 0, diffWorkspace) + diffCmdParallelism, 0, diffWorkspace, diffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(diffCmdKongStateFile) == 0 { @@ -51,6 +52,8 @@ that will be created, updated, or deleted. "any plugins associated with consumers") diffCmd.Flags().IntVar(&diffCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + diffCmd.Flags().BoolVar(&diffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") diffCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are diffed.\n"+ diff --git a/cmd/konnect_diff.go b/cmd/konnect_diff.go index a3d1c52eb..656f37c67 100644 --- a/cmd/konnect_diff.go +++ b/cmd/konnect_diff.go @@ -7,9 +7,10 @@ import ( ) var ( - konnectDiffCmdKongStateFile []string - konnectDiffCmdParallelism int - konnectDiffCmdNonZeroExitCode bool + konnectDiffCmdKongStateFile []string + konnectDiffCmdParallelism int + konnectDiffCmdNoMaskDeckEnvVarsValue bool + konnectDiffCmdNonZeroExitCode bool ) // newKonnectDiffCmd represents the 'deck konnect diff' command. @@ -32,7 +33,7 @@ func newKonnectDiffCmd() *cobra.Command { konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, true, - konnectDiffCmdParallelism) + konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -47,6 +48,8 @@ func newKonnectDiffCmd() *cobra.Command { "with consumers.") konnectDiffCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNonZeroExitCode, "non-zero-exit-code", false, "return exit code 2 if there is a diff present,\n"+ "exit code 0 if no diff is found,\n"+ diff --git a/cmd/konnect_sync.go b/cmd/konnect_sync.go index 200bd6b3d..11590f39b 100644 --- a/cmd/konnect_sync.go +++ b/cmd/konnect_sync.go @@ -24,7 +24,7 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, false, - konnectDiffCmdParallelism) + konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -39,6 +39,8 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, "with consumers.") konnectSyncCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectSyncCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") addSilenceEventsFlag(konnectSyncCmd.Flags()) return konnectSyncCmd } diff --git a/cmd/reset.go b/cmd/reset.go index 755eed088..a0690e0d7 100644 --- a/cmd/reset.go +++ b/cmd/reset.go @@ -9,9 +9,10 @@ import ( ) var ( - resetCmdForce bool - resetWorkspace string - resetAllWorkspaces bool + resetCmdForce bool + resetWorkspace string + resetAllWorkspaces bool + resetNoMaskDeckEnvVarsValue bool ) // newResetCmd represents the reset command @@ -47,7 +48,7 @@ By default, this command will ask for confirmation.`, mode := getMode(nil) if mode == modeKonnect { _ = sendAnalytics("reset", "", mode) - return resetKonnectV2(ctx) + return resetKonnectV2(ctx, resetNoMaskDeckEnvVarsValue) } rootClient, err := utils.GetKongClient(rootConfig) @@ -99,7 +100,7 @@ By default, this command will ask for confirmation.`, if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient, resetNoMaskDeckEnvVarsValue) if err != nil { return err } @@ -118,6 +119,8 @@ By default, this command will ask for confirmation.`, "(Kong Enterprise only).") resetCmd.Flags().BoolVar(&resetAllWorkspaces, "all-workspaces", false, "reset configuration of all workspaces (Kong Enterprise only).") + resetCmd.Flags().BoolVar(&resetNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") resetCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are deleted.\n"+ diff --git a/cmd/sync.go b/cmd/sync.go index 9c91bc40c..4da34c292 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -7,9 +7,10 @@ import ( ) var ( - syncCmdParallelism int - syncCmdDBUpdateDelay int - syncWorkspace string + syncCmdParallelism int + syncCmdDBUpdateDelay int + syncCmdNoMaskDeckEnvVarsValue bool + syncWorkspace string ) // newSyncCmd represents the sync command @@ -24,7 +25,7 @@ to get Kong's state in sync with the input state.`, Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), syncCmdKongStateFile, false, - syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace) + syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace, syncCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(syncCmdKongStateFile) == 0 { @@ -48,6 +49,8 @@ to get Kong's state in sync with the input state.`, "any plugins associated with consumers.") syncCmd.Flags().IntVar(&syncCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + syncCmd.Flags().BoolVar(&syncCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") syncCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are synced.\n"+ diff --git a/diff/diff.go b/diff/diff.go index 1c8c19381..4f7818f2f 100644 --- a/diff/diff.go +++ b/diff/diff.go @@ -322,8 +322,34 @@ type Stats struct { DeleteOps *utils.AtomicInt32Counter } +// Generete Diff output for 'sync' and 'diff' commands +func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, error) { + var diffString string + var err error + if oldObj, ok := e.OldObj.(*state.Document); ok { + if !isDelete { + diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) + } else { + diffString, err = getDocumentDiff(e.Obj.(*state.Document), oldObj) + } + } else { + if !isDelete { + diffString, err = getDiff(e.OldObj, e.Obj) + } else { + diffString, err = getDiff(e.Obj, e.OldObj) + } + } + if err != nil { + return "", err + } + if !noMaskValues { + diffString = maskEnvVarValue(diffString) + } + return diffString, err +} + // Solve generates a diff and walks the graph. -func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, []error) { +func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, noMaskValues bool) (Stats, []error) { stats := Stats{ CreateOps: &utils.AtomicInt32Counter{}, UpdateOps: &utils.AtomicInt32Counter{}, @@ -347,20 +373,23 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, c := e.Obj.(state.ConsoleString) switch e.Op { case crud.Create: - cprint.CreatePrintln("creating", e.Kind, c.Console()) - case crud.Update: - var diffString string - if oldObj, ok := e.OldObj.(*state.Document); ok { - diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) - } else { - diffString, err = getDiff(e.OldObj, e.Obj) + diffString, err := generateDiffString(e, false, noMaskValues) + if err != nil { + return nil, err } + cprint.CreatePrintln("creating", e.Kind, c.Console(), diffString) + case crud.Update: + diffString, err := generateDiffString(e, false, noMaskValues) if err != nil { return nil, err } cprint.UpdatePrintln("updating", e.Kind, c.Console(), diffString) case crud.Delete: - cprint.DeletePrintln("deleting", e.Kind, c.Console()) + diffString, err := generateDiffString(e, true, noMaskValues) + if err != nil { + return nil, err + } + cprint.DeletePrintln("deleting", e.Kind, c.Console(), diffString) default: panic("unknown operation " + e.Op.String()) } diff --git a/diff/diff_helpers.go b/diff/diff_helpers.go index e2eeb71ec..6bbd0c768 100644 --- a/diff/diff_helpers.go +++ b/diff/diff_helpers.go @@ -3,6 +3,9 @@ package diff import ( "encoding/json" "fmt" + "os" + "sort" + "strings" "github.com/Kong/gojsondiff" "github.com/Kong/gojsondiff/formatter" @@ -83,3 +86,35 @@ func getDiff(a, b interface{}) (string, error) { diffString, err := formatter.Format(d) return diffString, err } + +type EnvVar struct { + Key string + Value string +} + +func parseDeckEnvVars() []EnvVar { + const envVarPrefix = "DECK_" + var parsedEnvVars []EnvVar + + for _, envVarStr := range os.Environ() { + envPair := strings.SplitN(envVarStr, "=", 2) + if strings.HasPrefix(envPair[0], envVarPrefix) { + envVar := EnvVar{} + envVar.Key = envPair[0] + envVar.Value = envPair[1] + parsedEnvVars = append(parsedEnvVars, envVar) + } + } + + sort.Slice(parsedEnvVars, func(i, j int) bool { + return len(parsedEnvVars[i].Value) > len(parsedEnvVars[j].Value) + }) + return parsedEnvVars +} + +func maskEnvVarValue(diffString string) string { + for _, envVar := range parseDeckEnvVars() { + diffString = strings.Replace(diffString, envVar.Value, "[masked]", -1) + } + return diffString +} diff --git a/diff/diff_helpers_test.go b/diff/diff_helpers_test.go index 8f96e5446..453214e2a 100644 --- a/diff/diff_helpers_test.go +++ b/diff/diff_helpers_test.go @@ -1,6 +1,7 @@ package diff import ( + "os" "testing" "github.com/kong/deck/konnect" @@ -145,3 +146,35 @@ bar }) } } + +func Test_MaskEnvVarsValues(t *testing.T) { + tests := []struct { + name string + args string + want string + envVars map[string]string + }{ + { + name: "JSON", + envVars: map[string]string{ + "DECK_BAR": "barbar", + "DECK_BAZ": "bazbaz", + }, + args: `{"foo":"foo","bar":"barbar","baz":"bazbaz"}`, + want: `{"foo":"foo","bar":"[masked]","baz":"[masked]"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + if got := maskEnvVarValue(tt.args); got != tt.want { + t.Errorf("maskEnvVarValue() = %v\nwant %v", got, tt.want) + } + }) + } +} diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index a7664bc91..d0e086c70 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -3,6 +3,7 @@ package integration import ( + "os" "testing" "github.com/kong/deck/utils" @@ -10,53 +11,192 @@ import ( "github.com/stretchr/testify/assert" ) +var ( + expectedOutputMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "[masked]" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` + expectedOutputUnMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "mockbin.org" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` +) + // test scope: // - 1.x // - 2.x -func Test_Diff_Workspace_OlderThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_OlderThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + kong.RunWhenKong(t, "<3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_OlderThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } kong.RunWhenKong(t, "<3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } // test scope: // - 3.x -func Test_Diff_Workspace_NewerThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_NewerThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + kong.RunWhenKong(t, ">=3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_NewerThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } kong.RunWhenKong(t, ">=3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index 2761c4002..2a717445e 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -3,9 +3,11 @@ package integration import ( "context" + "io/ioutil" "os" "testing" + "github.com/fatih/color" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/kong/deck/cmd" @@ -137,12 +139,24 @@ func sync(kongFile string, opts ...string) error { return deckCmd.ExecuteContext(context.Background()) } -func diff(kongFile string, opts ...string) error { +func diff(kongFile string, opts ...string) (string, error) { deckCmd := cmd.NewRootCmd() args := []string{"diff", "-s", kongFile} if len(opts) > 0 { args = append(args, opts...) } deckCmd.SetArgs(args) - return deckCmd.ExecuteContext(context.Background()) + + // overwrite default standard output + r, w, _ := os.Pipe() + color.Output = w + + // execute decK command + cmdErr := deckCmd.ExecuteContext(context.Background()) + + // read command output + w.Close() + out, _ := ioutil.ReadAll(r) + + return string(out), cmdErr } diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml index 33b3a9cdf..dfd98b198 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml @@ -1,8 +1,10 @@ _workspace: test services: - name: svc1 - host: mockbin.org + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123 diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml index 328e12366..5462c612b 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml @@ -2,8 +2,10 @@ _format_version: "3.0" _workspace: test services: - name: svc1 - host: mockbin.org + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123