squash: use same format as plugins scoped to other entities

Makefile

@@ -47,6 +47,6 @@ setup-kong-ee:
 
 .PHONY: test-integration
 test-integration:
-	go test -v -tags=integration \
+	go test -v -count=1 -tags=integration \
 		-race \
 		./tests/integration/...

cmd/common.go

@@ -192,6 +192,10 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return err
 	}
 
+	if utils.Kong340Version.LTE(parsedKongVersion) {
+		dumpConfig.IsConsumerGroupScopedPluginSupported = true
+	}
+
 	// read the current state
 	var currentState *state.KongState
 	if workspaceExists {

cmd/reset.go

@@ -59,8 +59,16 @@ By default, this command will ask for confirmation.`,
 			if err != nil {
 				return fmt.Errorf("reading Kong version: %w", err)
 			}
+			parsedKongVersion, err := utils.ParseKongVersion(kongVersion)
+			if err != nil {
+				return fmt.Errorf("parsing Kong version: %w", err)
+			}
 			_ = sendAnalytics("reset", kongVersion, mode)
 
+			if utils.Kong340Version.LTE(parsedKongVersion) {
+				dumpConfig.IsConsumerGroupScopedPluginSupported = true
+			}
+
 			var workspaces []string
 			// Kong OSS or default workspace
 			if !resetAllWorkspaces && resetWorkspace == "" {

dump/dump.go

@@ -31,6 +31,9 @@ type Config struct {
 
 	// KonnectRuntimeGroup
 	KonnectRuntimeGroup string
+
+	// IsConsumerGroupScopedPluginSupported
+	IsConsumerGroupScopedPluginSupported bool
 }
 
 func deduplicate(stringSlice []string) []string {
@@ -205,6 +208,7 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 		plugins = excludeKonnectManagedPlugins(plugins)
 		if config.SkipConsumers {
 			plugins = excludeConsumersPlugins(plugins)
+			plugins = excludeConsumerGroupsPlugins(plugins)
 		}
 		state.Plugins = plugins
 		return nil
@@ -902,3 +906,15 @@ func excludeConsumersPlugins(plugins []*kong.Plugin) []*kong.Plugin {
 	}
 	return filtered
 }
+
+// excludeConsumerGroupsPlugins filter out consumer-groups plugins
+func excludeConsumerGroupsPlugins(plugins []*kong.Plugin) []*kong.Plugin {
+	var filtered []*kong.Plugin
+	for _, p := range plugins {
+		if p.ConsumerGroup != nil && !utils.Empty(p.ConsumerGroup.ID) {
+			continue
+		}
+		filtered = append(filtered, p)
+	}
+	return filtered
+}

file/builder.go

@@ -12,6 +12,8 @@ import (
 	"github.com/kong/go-kong/kong"
 )
 
+const ratelimitingAdvancedPluginName = "rate-limiting-advanced"
+
 type stateBuilder struct {
 	targetContent   *Content
 	rawState        *utils.KongRawState
@@ -35,6 +37,8 @@ type stateBuilder struct {
 
 	checkRoutePaths bool
 
+	isConsumerGroupScopedPluginSupported bool
+
 	err error
 }
 
@@ -69,6 +73,10 @@ func (b *stateBuilder) build() (*utils.KongRawState, *utils.KonnectRawState, err
 		b.checkRoutePaths = true
 	}
 
+	if utils.Kong340Version.LTE(b.kongVersion) {
+		b.isConsumerGroupScopedPluginSupported = true
+	}
+
 	// build
 	b.certificates()
 	if !b.skipCACerts {
@@ -122,22 +130,44 @@ func (b *stateBuilder) consumerGroups() {
 			return
 		}
 
-		for _, plugin := range cg.Plugins {
-			if utils.Empty(plugin.ID) {
-				current, err := b.currentState.ConsumerGroupPlugins.Get(
-					*plugin.Name, *cg.ConsumerGroup.ID,
-				)
-				if errors.Is(err, state.ErrNotFound) {
-					plugin.ID = uuid()
-				} else if err != nil {
-					b.err = err
-					return
-				} else {
-					plugin.ID = kong.String(*current.ID)
+		if b.isConsumerGroupScopedPluginSupported {
+			var plugins []FPlugin
+			for _, plugin := range cg.Plugins {
+				plugin.ConsumerGroup = utils.GetConsumerGroupReference(cg.ConsumerGroup)
+				plugins = append(plugins, FPlugin{
+					Plugin: kong.Plugin{
+						ID:     plugin.ID,
+						Name:   plugin.Name,
+						Config: plugin.Config,
+						ConsumerGroup: &kong.ConsumerGroup{
+							ID: cg.ID,
+						},
+					},
+				})
+			}
+
+			if err := b.ingestPlugins(plugins); err != nil {
+				b.err = err
+				return
+			}
+		} else {
+			for _, plugin := range cg.Plugins {
+				if utils.Empty(plugin.ID) {
+					current, err := b.currentState.ConsumerGroupPlugins.Get(
+						*plugin.Name, *cg.ConsumerGroup.ID,
+					)
+					if errors.Is(err, state.ErrNotFound) {
+						plugin.ID = uuid()
+					} else if err != nil {
+						b.err = err
+						return
+					} else {
+						plugin.ID = kong.String(*current.ID)
+					}
 				}
+				b.defaulter.MustSet(plugin)
+				cgo.Plugins = append(cgo.Plugins, plugin)
 			}
-			b.defaulter.MustSet(plugin)
-			cgo.Plugins = append(cgo.Plugins, plugin)
 		}
 		b.rawState.ConsumerGroups = append(b.rawState.ConsumerGroups, &cgo)
 	}
@@ -900,6 +930,19 @@ func (b *stateBuilder) plugins() {
 			}
 			p.ConsumerGroup = utils.GetConsumerGroupReference(cg.ConsumerGroup)
 		}
+
+		if b.isConsumerGroupScopedPluginSupported && *p.Name == ratelimitingAdvancedPluginName {
+			// check if deprecated consumer-groups configuration is present in the config
+			_, consumerGroupsFound := p.Config["consumer_groups"]
+			_, enforcedConsumerGroupFound := p.Config["enforced_consumer_group"]
+			if consumerGroupsFound || enforcedConsumerGroupFound {
+				b.err = errors.New("a rate-limiting-advanced plugin with config.consumer_groups\n" +
+					"and/or config.enforced_consumer_group was found. Please use Consumer Groups scoped\n" +
+					"Plugins when running against Kong Enterprise 3.4.0 and above.\n\n" +
+					"Check DOC_LINK for more information")
+				return
+			}
+		}
 		plugins = append(plugins, p)
 	}
 	if err := b.ingestPlugins(plugins); err != nil {

file/writer.go

@@ -415,6 +415,18 @@ func populatePlugins(kongState *state.KongState, file *Content,
 			}
 			p.Route.ID = &rID
 		}
+		if p.ConsumerGroup != nil {
+			associations++
+			cgID := *p.ConsumerGroup.ID
+			cg, err := kongState.ConsumerGroups.Get(cgID)
+			if err != nil {
+				return fmt.Errorf("unable to get consumer-group %s for plugin %s [%s]: %w", cgID, *p.Name, *p.ID, err)
+			}
+			if !utils.Empty(cg.Name) {
+				cgID = *cg.Name
+			}
+			p.ConsumerGroup.ID = &cgID
+		}
 		if associations == 0 || associations > 1 {
 			utils.ZeroOutID(p, p.Name, config.WithID)
 			utils.ZeroOutTimestamps(p)
@@ -703,13 +715,13 @@ func populateConsumerGroups(kongState *state.KongState, file *Content,
 	if err != nil {
 		return err
 	}
-	plugins, err := kongState.ConsumerGroupPlugins.GetAll()
+	cgPlugins, err := kongState.ConsumerGroupPlugins.GetAll()
 	if err != nil {
 		return err
 	}
 	for _, cg := range consumerGroups {
 		group := FConsumerGroupObject{ConsumerGroup: cg.ConsumerGroup}
-		for _, plugin := range plugins {
+		for _, plugin := range cgPlugins {
 			if plugin.ID != nil && cg.ID != nil {
 				if plugin.ConsumerGroup != nil && *plugin.ConsumerGroup.ID == *cg.ID {
 					utils.ZeroOutID(plugin, plugin.Name, config.WithID)
@@ -720,6 +732,25 @@ func populateConsumerGroups(kongState *state.KongState, file *Content,
 				}
 			}
 		}
+
+		plugins, err := kongState.Plugins.GetAllByConsumerGroupID(*cg.ID)
+		if err != nil {
+			return err
+		}
+		for _, plugin := range plugins {
+			if plugin.ID != nil && cg.ID != nil {
+				if plugin.ConsumerGroup != nil && *plugin.ConsumerGroup.ID == *cg.ID {
+					utils.ZeroOutID(plugin, plugin.Name, config.WithID)
+					utils.ZeroOutID(plugin.ConsumerGroup, plugin.ConsumerGroup.Name, config.WithID)
+					group.Plugins = append(group.Plugins, &kong.ConsumerGroupPlugin{
+						ID:     plugin.ID,
+						Name:   plugin.Name,
+						Config: plugin.Config,
+					})
+				}
+			}
+		}
+
 		utils.ZeroOutID(&group, group.Name, config.WithID)
 		utils.ZeroOutTimestamps(&group)
 		file.ConsumerGroups = append(file.ConsumerGroups, group)

state/plugin.go

@@ -300,6 +300,14 @@ func (k *PluginsCollection) GetAllByConsumerID(id string) ([]*Plugin,
 	return k.getAllPluginsBy(pluginsByConsumerID, id)
 }
 
+// GetAllByConsumerGroupID returns all plugins referencing a consumer-group
+// by its id.
+func (k *PluginsCollection) GetAllByConsumerGroupID(id string) ([]*Plugin,
+	error,
+) {
+	return k.getAllPluginsBy(pluginsByConsumerGroupID, id)
+}
+
 // Update updates a plugin
 func (k *PluginsCollection) Update(plugin Plugin) error {
 	// TODO abstract this check in the go-memdb library itself

tests/integration/dump_test.go

@@ -98,7 +98,58 @@ func Test_Dump_SkipConsumers(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			runWhen(t, "enterprise", ">=3.2.0")
+			runWhen(t, "enterprise", ">=3.2.0 <3.4.0")
+			teardown := setup(t)
+			defer teardown(t)
+
+			assert.NoError(t, sync(tc.stateFile))
+
+			var (
+				output string
+				err    error
+			)
+			if tc.skipConsumers {
+				output, err = dump(
+					"--skip-consumers",
+					"-o", "-",
+				)
+			} else {
+				output, err = dump(
+					"-o", "-",
+				)
+			}
+			assert.NoError(t, err)
+
+			expected, err := readFile(tc.expectedFile)
+			assert.NoError(t, err)
+			assert.Equal(t, output, expected)
+		})
+	}
+}
+
+func Test_Dump_SkipConsumers_After340(t *testing.T) {
+	tests := []struct {
+		name          string
+		stateFile     string
+		expectedFile  string
+		skipConsumers bool
+	}{
+		{
+			name:          "dump with skip-consumers",
+			stateFile:     "testdata/dump/002-skip-consumers/kong34.yaml",
+			expectedFile:  "testdata/dump/002-skip-consumers/expected.yaml",
+			skipConsumers: true,
+		},
+		{
+			name:          "dump with no skip-consumers",
+			stateFile:     "testdata/dump/002-skip-consumers/kong34.yaml",
+			expectedFile:  "testdata/dump/002-skip-consumers/expected-no-skip-34.yaml",
+			skipConsumers: false,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			runWhen(t, "enterprise", ">=3.4.0")
 			teardown := setup(t)
 			defer teardown(t)