updates

dump/dump.go

@@ -238,6 +238,9 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 
 	group.Go(func() error {
 		consumerGroups, err := GetAllConsumerGroups(ctx, client, config.SelectorTags)
+		if kong.IsNotFoundErr(err) {
+			return nil
+		}
 		if err != nil {
 			return fmt.Errorf("consumer_groups: %w", err)
 		}

file/builder.go

@@ -77,8 +77,8 @@ func (b *stateBuilder) build() (*utils.KongRawState, *utils.KonnectRawState, err
 	b.services()
 	b.routes()
 	b.upstreams()
-	b.consumers()
 	b.consumerGroups()
+	b.consumers()
 	b.plugins()
 	b.enterprise()
 
@@ -116,21 +116,6 @@ func (b *stateBuilder) consumerGroups() {
 			ConsumerGroup: &cg.ConsumerGroup,
 		}
 
-		for _, consumer := range cg.Consumers {
-			if utils.Empty(consumer.ID) {
-				current, err := b.currentState.Consumers.Get(*consumer.Username)
-				if err == state.ErrNotFound {
-					consumer.ID = uuid()
-				} else if err != nil {
-					b.err = err
-					return
-				} else {
-					consumer.ID = kong.String(*current.ID)
-				}
-			}
-			cgo.Consumers = append(cgo.Consumers, consumer)
-		}
-
 		for _, plugin := range cg.Plugins {
 			if utils.Empty(plugin.ID) {
 				current, err := b.currentState.ConsumerGroupPlugins.Get(
@@ -275,6 +260,12 @@ func (b *stateBuilder) consumers() {
 			return
 		}
 
+		// ingest consumer into consumer group
+		if err := b.ingestIntoConsumerGroup(c); err != nil {
+			b.err = err
+			return
+		}
+
 		// plugins for the Consumer
 		var plugins []FPlugin
 		for _, p := range c.Plugins {
@@ -356,6 +347,37 @@ func (b *stateBuilder) consumers() {
 	}
 }
 
+func (b *stateBuilder) ingestIntoConsumerGroup(consumer FConsumer) error {
+	for _, group := range consumer.Groups {
+		found := false
+		for _, cg := range b.rawState.ConsumerGroups {
+			if group.ID != nil && *cg.ConsumerGroup.ID == *group.ID {
+				cg.Consumers = append(cg.Consumers, &consumer.Consumer)
+				found = true
+				break
+
+			}
+			if group.Name != nil && *cg.ConsumerGroup.Name == *group.Name {
+				cg.Consumers = append(cg.Consumers, &consumer.Consumer)
+				found = true
+				break
+			}
+		}
+		if !found {
+			var groupIdentifier string
+			if group.Name != nil {
+				groupIdentifier = *group.Name
+			} else {
+				groupIdentifier = *group.ID
+			}
+			return fmt.Errorf(
+				"consumer-group '%s' not found for consumer '%s'", groupIdentifier, *consumer.ID,
+			)
+		}
+	}
+	return nil
+}
+
 func (b *stateBuilder) ingestKeyAuths(creds []kong.KeyAuth) error {
 	for _, cred := range creds {
 		cred := cred

file/kong_json_schema.json

@@ -314,7 +314,6 @@
           "type": "object"
         },
         "consumer_group": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
           "$ref": "#/definitions/ConsumerGroup"
         },
         "created_at": {
@@ -419,6 +418,13 @@
         "custom_id": {
           "type": "string"
         },
+        "groups": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ConsumerGroup"
+          },
+          "type": "array"
+        },
         "hmacauth_credentials": {
           "items": {
             "$schema": "http://json-schema.org/draft-04/schema#",

file/types.go

@@ -493,6 +493,7 @@ type FConsumer struct {
 	Oauth2Creds   []*kong.Oauth2Credential `json:"oauth2_credentials,omitempty" yaml:"oauth2_credentials,omitempty"`
 	ACLGroups     []*kong.ACLGroup         `json:"acls,omitempty" yaml:"acls,omitempty"`
 	MTLSAuths     []*kong.MTLSAuth         `json:"mtls_auth_credentials,omitempty" yaml:"mtls_auth_credentials,omitempty"`
+	Groups        []*kong.ConsumerGroup    `json:"groups,omitempty" yaml:"groups,omitempty"`
 }
 
 // sortKey is used for sorting.

file/writer.go

@@ -546,6 +546,10 @@ func populateConsumers(kongState *state.KongState, file *Content,
 	if err != nil {
 		return err
 	}
+	consumerGroups, err := kongState.ConsumerGroups.GetAll()
+	if err != nil {
+		return err
+	}
 	for _, c := range consumers {
 		c := FConsumer{Consumer: c.Consumer}
 		plugins, err := kongState.Plugins.GetAllByConsumerID(*c.ID)
@@ -642,6 +646,24 @@ func populateConsumers(kongState *state.KongState, file *Content,
 			k.Consumer = nil
 			c.MTLSAuths = append(c.MTLSAuths, &k.MTLSAuth)
 		}
+		// populate groups
+		for _, cg := range consumerGroups {
+			cg := *cg
+			_, err := kongState.ConsumerGroupConsumers.Get(*c.ID, *cg.ID)
+			if err != nil {
+				if err != state.ErrNotFound {
+					return err
+				}
+				continue
+			}
+			utils.ZeroOutID(&cg, cg.Name, config.WithID)
+			utils.ZeroOutTimestamps(&cg)
+			utils.MustRemoveTags(&cg.ConsumerGroup, config.SelectTags)
+			c.Groups = append(c.Groups, cg.DeepCopy())
+		}
+		sort.SliceStable(c.Plugins, func(i, j int) bool {
+			return compareOrder(c.Plugins[i], c.Plugins[j])
+		})
 		utils.ZeroOutID(&c, c.Username, config.WithID)
 		utils.ZeroOutTimestamps(&c)
 		utils.MustRemoveTags(&c.Consumer, config.SelectTags)
@@ -680,31 +702,19 @@ func populateConsumerGroups(kongState *state.KongState, file *Content,
 	if err != nil {
 		return err
 	}
-	consumers, err := kongState.ConsumerGroupConsumers.GetAll()
-	if err != nil {
-		return err
-	}
 	plugins, err := kongState.ConsumerGroupPlugins.GetAll()
 	if err != nil {
 		return err
 	}
 	for _, cg := range consumerGroups {
 		group := FConsumerGroupObject{ConsumerGroup: cg.ConsumerGroup}
-		for _, consumer := range consumers {
-			if consumer.ConsumerGroup.ID != nil && cg.ID != nil {
-				if *consumer.ConsumerGroup.ID == *cg.ID {
-					utils.ZeroOutID(consumer.Consumer, consumer.Consumer.Username, config.WithID)
-					utils.ZeroOutTimestamps(consumer.Consumer)
-					group.Consumers = append(group.Consumers, consumer.Consumer)
-				}
-			}
-		}
 		for _, plugin := range plugins {
 			if plugin.ID != nil && cg.ID != nil {
 				if *plugin.ConsumerGroup.ID == *cg.ID {
 					utils.ZeroOutID(plugin, plugin.Name, config.WithID)
 					utils.ZeroOutID(plugin.ConsumerGroup, plugin.ConsumerGroup.Name, config.WithID)
 					utils.ZeroOutTimestamps(plugin.ConsumerGroupPlugin.ConsumerGroup)
+					utils.ZeroOutField(&plugin.ConsumerGroupPlugin, "ConsumerGroup")
 					group.Plugins = append(group.Plugins, &plugin.ConsumerGroupPlugin)
 				}
 			}

tests/integration/sync_test.go

@@ -2458,15 +2458,15 @@ func Test_Sync_ConsumerGroupsTill30(t *testing.T) {
 	}{
 		{
 			name:     "creates consumer groups",
-			kongFile: "testdata/sync/012-consumer-groups/kong.yaml",
+			kongFile: "testdata/sync/015-consumer-groups/kong.yaml",
 			expectedState: utils.KongRawState{
 				Consumers:      consumerGroupsConsumers,
 				ConsumerGroups: []*kong.ConsumerGroupObject{},
 			},
 		},
 		{
 			name:     "creates consumer groups",
-			kongFile: "testdata/sync/013-consumer-groups-and-plugins/kong.yaml",
+			kongFile: "testdata/sync/016-consumer-groups-and-plugins/kong.yaml",
 			expectedState: utils.KongRawState{
 				Consumers:      consumerGroupsConsumers,
 				ConsumerGroups: consumerGroupsWithRLA,
@@ -2500,15 +2500,15 @@ func Test_Sync_ConsumerGroupsFrom30(t *testing.T) {
 	}{
 		{
 			name:     "creates consumer groups",
-			kongFile: "testdata/sync/012-consumer-groups/kong3x.yaml",
+			kongFile: "testdata/sync/015-consumer-groups/kong3x.yaml",
 			expectedState: utils.KongRawState{
 				Consumers:      consumerGroupsConsumers,
 				ConsumerGroups: consumerGroups,
 			},
 		},
 		{
 			name:     "creates consumer groups",
-			kongFile: "testdata/sync/013-consumer-groups-and-plugins/kong3x.yaml",
+			kongFile: "testdata/sync/016-consumer-groups-and-plugins/kong3x.yaml",
 			expectedState: utils.KongRawState{
 				Consumers:      consumerGroupsConsumers,
 				ConsumerGroups: consumerGroupsWithRLA,

tests/integration/testdata/sync/012-consumer-groups/kong3x.yaml → tests/integration/testdata/sync/015-consumer-groups/kong3x.yaml

@@ -1,13 +1,14 @@
 _format_version: "3.0"
 consumer_groups:
 - name: silver
-  consumers:
-  - username: bar
-  - username: baz
 - name: gold
-  consumers:
-  - username: foo
 consumers:
 - username: foo
+  groups:
+    - name: gold
 - username: bar
+  groups:
+    - name: silver
 - username: baz
+  groups:
+    - name: gold

utils/zero.go

@@ -6,7 +6,7 @@ import (
 
 var zero reflect.Value
 
-func zeroOutField(obj interface{}, field string) {
+func ZeroOutField(obj interface{}, field string) {
 	ptr := reflect.ValueOf(obj)
 	if ptr.Kind() != reflect.Ptr {
 		return
@@ -29,10 +29,10 @@ func ZeroOutID(obj interface{}, altName *string, withID bool) {
 		return
 	}
 	// zero the ID field
-	zeroOutField(obj, "ID")
+	ZeroOutField(obj, "ID")
 }
 
 func ZeroOutTimestamps(obj interface{}) {
-	zeroOutField(obj, "CreatedAt")
-	zeroOutField(obj, "UpdatedAt")
+	ZeroOutField(obj, "CreatedAt")
+	ZeroOutField(obj, "UpdatedAt")
 }