diff --git a/cmd/common.go b/cmd/common.go index ab9160601..d0f5ece26 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -27,8 +27,9 @@ const ( ) var ( - dumpConfig dump.Config - assumeYes bool + dumpConfig dump.Config + assumeYes bool + noMaskValues bool ) type mode int @@ -280,6 +281,7 @@ func performDiff(ctx context.Context, currentState, targetState *state.KongState TargetState: targetState, KongClient: client, StageDelaySec: delay, + NoMaskValues: noMaskValues, }) if err != nil { return 0, err diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go index bc9c5e032..9a5887213 100644 --- a/cmd/common_konnect.go +++ b/cmd/common_konnect.go @@ -244,6 +244,7 @@ func syncKonnect(ctx context.Context, TargetState: targetState, KongClient: kongClient, KonnectClient: konnectClient, + NoMaskValues: noMaskValues, }) if err != nil { return err diff --git a/cmd/diff.go b/cmd/diff.go index d6055b68c..8bb66ff92 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -51,6 +51,8 @@ that will be created, updated, or deleted. "any plugins associated with consumers") diffCmd.Flags().IntVar(&diffCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + diffCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") diffCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are diffed.\n"+ diff --git a/cmd/konnect_diff.go b/cmd/konnect_diff.go index a3d1c52eb..b5cf55ade 100644 --- a/cmd/konnect_diff.go +++ b/cmd/konnect_diff.go @@ -47,6 +47,8 @@ func newKonnectDiffCmd() *cobra.Command { "with consumers.") konnectDiffCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectDiffCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNonZeroExitCode, "non-zero-exit-code", false, "return exit code 2 if there is a diff present,\n"+ "exit code 0 if no diff is found,\n"+ diff --git a/cmd/konnect_sync.go b/cmd/konnect_sync.go index 200bd6b3d..2bfd9d23c 100644 --- a/cmd/konnect_sync.go +++ b/cmd/konnect_sync.go @@ -39,6 +39,8 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, "with consumers.") konnectSyncCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectSyncCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") addSilenceEventsFlag(konnectSyncCmd.Flags()) return konnectSyncCmd } diff --git a/cmd/reset.go b/cmd/reset.go index 755eed088..203a68ac8 100644 --- a/cmd/reset.go +++ b/cmd/reset.go @@ -118,6 +118,8 @@ By default, this command will ask for confirmation.`, "(Kong Enterprise only).") resetCmd.Flags().BoolVar(&resetAllWorkspaces, "all-workspaces", false, "reset configuration of all workspaces (Kong Enterprise only).") + resetCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") resetCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are deleted.\n"+ diff --git a/cmd/sync.go b/cmd/sync.go index 9c91bc40c..c35895327 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -48,6 +48,8 @@ to get Kong's state in sync with the input state.`, "any plugins associated with consumers.") syncCmd.Flags().IntVar(&syncCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + syncCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") syncCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are synced.\n"+ diff --git a/diff/diff.go b/diff/diff.go index 1c8c19381..a90224c63 100644 --- a/diff/diff.go +++ b/diff/diff.go @@ -55,6 +55,8 @@ type Syncer struct { konnectClient *konnect.Client entityDiffers map[types.EntityType]types.Differ + + noMaskValues bool } type SyncerOpts struct { @@ -66,6 +68,8 @@ type SyncerOpts struct { SilenceWarnings bool StageDelaySec int + + NoMaskValues bool } // NewSyncer constructs a Syncer. @@ -79,6 +83,8 @@ func NewSyncer(opts SyncerOpts) (*Syncer, error) { silenceWarnings: opts.SilenceWarnings, stageDelaySec: opts.StageDelaySec, + + noMaskValues: opts.NoMaskValues, } err := s.init() @@ -322,6 +328,32 @@ type Stats struct { DeleteOps *utils.AtomicInt32Counter } +// Generete Diff output for 'sync' and 'diff' commands +func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, error) { + var diffString string + var err error + if oldObj, ok := e.OldObj.(*state.Document); ok { + if !isDelete { + diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) + } else { + diffString, err = getDocumentDiff(e.Obj.(*state.Document), oldObj) + } + } else { + if !isDelete { + diffString, err = getDiff(e.OldObj, e.Obj) + } else { + diffString, err = getDiff(e.Obj, e.OldObj) + } + } + if err != nil { + return "", err + } + if !noMaskValues { + diffString = maskEnvVarValue(diffString) + } + return diffString, err +} + // Solve generates a diff and walks the graph. func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, []error) { stats := Stats{ @@ -347,20 +379,23 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, c := e.Obj.(state.ConsoleString) switch e.Op { case crud.Create: - cprint.CreatePrintln("creating", e.Kind, c.Console()) - case crud.Update: - var diffString string - if oldObj, ok := e.OldObj.(*state.Document); ok { - diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) - } else { - diffString, err = getDiff(e.OldObj, e.Obj) + diffString, err := generateDiffString(e, false, sc.noMaskValues) + if err != nil { + return nil, err } + cprint.CreatePrintln("creating", e.Kind, c.Console(), diffString) + case crud.Update: + diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err } cprint.UpdatePrintln("updating", e.Kind, c.Console(), diffString) case crud.Delete: - cprint.DeletePrintln("deleting", e.Kind, c.Console()) + diffString, err := generateDiffString(e, true, sc.noMaskValues) + if err != nil { + return nil, err + } + cprint.DeletePrintln("deleting", e.Kind, c.Console(), diffString) default: panic("unknown operation " + e.Op.String()) } diff --git a/diff/diff_helpers.go b/diff/diff_helpers.go index e2eeb71ec..6bbd0c768 100644 --- a/diff/diff_helpers.go +++ b/diff/diff_helpers.go @@ -3,6 +3,9 @@ package diff import ( "encoding/json" "fmt" + "os" + "sort" + "strings" "github.com/Kong/gojsondiff" "github.com/Kong/gojsondiff/formatter" @@ -83,3 +86,35 @@ func getDiff(a, b interface{}) (string, error) { diffString, err := formatter.Format(d) return diffString, err } + +type EnvVar struct { + Key string + Value string +} + +func parseDeckEnvVars() []EnvVar { + const envVarPrefix = "DECK_" + var parsedEnvVars []EnvVar + + for _, envVarStr := range os.Environ() { + envPair := strings.SplitN(envVarStr, "=", 2) + if strings.HasPrefix(envPair[0], envVarPrefix) { + envVar := EnvVar{} + envVar.Key = envPair[0] + envVar.Value = envPair[1] + parsedEnvVars = append(parsedEnvVars, envVar) + } + } + + sort.Slice(parsedEnvVars, func(i, j int) bool { + return len(parsedEnvVars[i].Value) > len(parsedEnvVars[j].Value) + }) + return parsedEnvVars +} + +func maskEnvVarValue(diffString string) string { + for _, envVar := range parseDeckEnvVars() { + diffString = strings.Replace(diffString, envVar.Value, "[masked]", -1) + } + return diffString +} diff --git a/diff/diff_helpers_test.go b/diff/diff_helpers_test.go index 8f96e5446..453214e2a 100644 --- a/diff/diff_helpers_test.go +++ b/diff/diff_helpers_test.go @@ -1,6 +1,7 @@ package diff import ( + "os" "testing" "github.com/kong/deck/konnect" @@ -145,3 +146,35 @@ bar }) } } + +func Test_MaskEnvVarsValues(t *testing.T) { + tests := []struct { + name string + args string + want string + envVars map[string]string + }{ + { + name: "JSON", + envVars: map[string]string{ + "DECK_BAR": "barbar", + "DECK_BAZ": "bazbaz", + }, + args: `{"foo":"foo","bar":"barbar","baz":"bazbaz"}`, + want: `{"foo":"foo","bar":"[masked]","baz":"[masked]"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + if got := maskEnvVarValue(tt.args); got != tt.want { + t.Errorf("maskEnvVarValue() = %v\nwant %v", got, tt.want) + } + }) + } +} diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index 4aca76c82..4af042af1 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -3,59 +3,214 @@ package integration import ( + "os" "testing" "github.com/kong/deck/utils" "github.com/stretchr/testify/assert" ) +var ( + expectedOutputMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "[masked]" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "tags": [ ++ "[masked] is an external host. I like [masked]!", ++ "foo:foo", ++ "baz:[masked]", ++ "another:[masked]", ++ "bar:[masked]" ++ ] ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` + + expectedOutputUnMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "mockbin.org" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "tags": [ ++ "mockbin.org is an external host. I like mockbin.org!", ++ "foo:foo", ++ "baz:bazbaz", ++ "another:bazbaz", ++ "bar:barbar" ++ ] ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` + + diffEnvVars = map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + "DECK_BARR": "barbar", + "DECK_BAZZ": "bazbaz", // used more than once + "DECK_FUB": "fubfub", // unused + "DECK_FOO": "foo_test", // unused, partial match + } +) + // test scope: // - 1.x // - 2.x -func Test_Diff_Workspace_OlderThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_OlderThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: diffEnvVars, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } runWhen(t, "kong", "<3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_OlderThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: diffEnvVars, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + runWhen(t, "kong", "<3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } // test scope: // - 3.x -func Test_Diff_Workspace_NewerThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_NewerThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: diffEnvVars, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + runWhen(t, "kong", ">=3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_NewerThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: diffEnvVars, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } runWhen(t, "kong", ">=3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index f0417417a..b03298987 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -3,9 +3,11 @@ package integration import ( "context" + "io/ioutil" "os" "testing" + "github.com/fatih/color" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/kong/deck/cmd" @@ -173,12 +175,24 @@ func sync(kongFile string, opts ...string) error { return deckCmd.ExecuteContext(context.Background()) } -func diff(kongFile string, opts ...string) error { +func diff(kongFile string, opts ...string) (string, error) { deckCmd := cmd.NewRootCmd() args := []string{"diff", "-s", kongFile} if len(opts) > 0 { args = append(args, opts...) } deckCmd.SetArgs(args) - return deckCmd.ExecuteContext(context.Background()) + + // overwrite default standard output + r, w, _ := os.Pipe() + color.Output = w + + // execute decK command + cmdErr := deckCmd.ExecuteContext(context.Background()) + + // read command output + w.Close() + out, _ := ioutil.ReadAll(r) + + return string(out), cmdErr } diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml index 33b3a9cdf..01b37490e 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml @@ -1,8 +1,16 @@ _workspace: test services: -- name: svc1 - host: mockbin.org + - name: svc1 + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} + tags: + - ${{ env "DECK_SVC1_HOSTNAME" }} is an external host. I like mockbin.org! + - foo:foo + - baz:${{ env "DECK_BAZZ" }} + - another:${{ env "DECK_BAZZ" }} + - bar:${{ env "DECK_BARR" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123 diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml index 328e12366..84c9692d0 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml @@ -1,9 +1,17 @@ _format_version: "3.0" _workspace: test services: -- name: svc1 - host: mockbin.org + - name: svc1 + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} + tags: + - ${{ env "DECK_SVC1_HOSTNAME" }} is an external host. I like mockbin.org! + - foo:foo + - baz:${{ env "DECK_BAZZ" }} + - another:${{ env "DECK_BAZZ" }} + - bar:${{ env "DECK_BARR" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123