elasticache: Allow for tagging retries

.changelog/36310.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+resource/aws_elasticache_replication_group: Fix bugs causing errors like `InvalidReplicationGroupState: Cluster not in available state to perform tagging operations.`
+```

internal/generate/tags/main.go

@@ -54,6 +54,10 @@ var (
 	listTagsOp                 = flag.String("ListTagsOp", "ListTagsForResource", "listTagsOp")
 	listTagsOpPaginated        = flag.Bool("ListTagsOpPaginated", false, "whether ListTagsOp is paginated")
 	listTagsOutTagsElem        = flag.String("ListTagsOutTagsElem", "Tags", "listTagsOutTagsElem")
+	retryTagsListTagsType      = flag.String("RetryTagsListTagsType", "", "type of the first ListTagsOp return value such as TagListMessage")
+	retryTagsErrorCodes        = flag.String("RetryTagsErrorCodes", "", "comma-separated list of error codes to retry, must be used with RetryTagsListTagsType and same length as RetryTagsErrorMessages")
+	retryTagsErrorMessages     = flag.String("RetryTagsErrorMessages", "", "comma-separated list of error messages to retry, must be used with RetryTagsListTagsType and same length as RetryTagsErrorCodes")
+	retryTagsTimeout           = flag.Duration("RetryTagsTimeout", 1*time.Minute, "Timeout for retrying tag operations")
 	setTagsOutFunc             = flag.String("SetTagsOutFunc", "setTagsOut", "setTagsOutFunc")
 	tagInCustomVal             = flag.String("TagInCustomVal", "", "tagInCustomVal")
 	tagInIDElem                = flag.String("TagInIDElem", "ResourceArn", "tagInIDElem")
@@ -171,7 +175,11 @@ type TemplateData struct {
 	ListTagsOutTagsElem        string
 	ParentNotFoundErrCode      string
 	ParentNotFoundErrMsg       string
-	RetryCreateOnNotFound      string
+	RetryCreateOnNotFound      string // is this used?
+	RetryTagsListTagsType      string
+	RetryTagsErrorCodes        []string
+	RetryTagsErrorMessages     []string
+	RetryTagsTimeout           string
 	ServiceTagsMap             bool
 	SetTagsOutFunc             string
 	TagInCustomVal             string
@@ -295,6 +303,15 @@ func main() {
 		}
 	}
 
+	var cleanRetryErrorCodes []string
+	for _, c := range strings.Split(*retryTagsErrorCodes, ",") {
+		if strings.HasPrefix(c, fmt.Sprintf("%s.", servicePackage)) || strings.HasPrefix(c, "types.") {
+			cleanRetryErrorCodes = append(cleanRetryErrorCodes, c)
+		} else {
+			cleanRetryErrorCodes = append(cleanRetryErrorCodes, fmt.Sprintf(`"%s"`, c))
+		}
+	}
+
 	templateData := TemplateData{
 		AWSService:             awsPkg,
 		AWSServiceIfacePackage: awsIntfPkg,
@@ -312,8 +329,8 @@ func main() {
 		SkipServiceImp:    *skipServiceImp,
 		SkipTypesImp:      *skipTypesImp,
 		TfLogPkg:          *updateTags,
-		TfResourcePkg:     (*getTag || *waitForPropagation),
-		TimePkg:           *waitForPropagation,
+		TfResourcePkg:     *getTag || *waitForPropagation || *retryTagsListTagsType != "",
+		TimePkg:           *waitForPropagation || *retryTagsListTagsType != "",
 
 		CreateTagsFunc:             createTagsFunc,
 		GetTagFunc:                 *getTagFunc,
@@ -329,6 +346,10 @@ func main() {
 		ListTagsOutTagsElem:        *listTagsOutTagsElem,
 		ParentNotFoundErrCode:      *parentNotFoundErrCode,
 		ParentNotFoundErrMsg:       *parentNotFoundErrMsg,
+		RetryTagsListTagsType:      *retryTagsListTagsType,
+		RetryTagsErrorCodes:        cleanRetryErrorCodes,
+		RetryTagsErrorMessages:     strings.Split(*retryTagsErrorMessages, ","),
+		RetryTagsTimeout:           formatDuration(*retryTagsTimeout),
 		ServiceTagsMap:             *serviceTagsMap,
 		SetTagsOutFunc:             *setTagsOutFunc,
 		TagInCustomVal:             *tagInCustomVal,

internal/generate/tags/templates/v1/get_tag_body.tmpl

@@ -23,7 +23,25 @@ func {{ .GetTagFunc }}(ctx context.Context, conn {{ .ClientType }}, identifier{{
 		},
 	}
 
+	{{ if .RetryTagsListTagsType }}
+	output, err := tfresource.RetryGWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+		func() (*{{ .TagPackage  }}.{{ .RetryTagsListTagsType }}, error) {
+			return conn.{{ .ListTagsOp }}WithContext(ctx, input)
+		},
+		[]string{
+			{{- range .RetryTagsErrorCodes }}
+			{{ . }},
+			{{- end }}
+		},
+		[]string{
+			{{- range .RetryTagsErrorMessages }}
+			"{{ . }}",
+			{{- end }}
+		},
+	)
+	{{ else }}
 	output, err := conn.{{ .ListTagsOp }}WithContext(ctx, input)
+	{{- end }}
 
 	if err != nil {
 		return nil, err

internal/generate/tags/templates/v1/list_tags_body.tmpl

@@ -28,6 +28,39 @@ func {{ .ListTagsFunc }}(ctx context.Context, conn {{ .ClientType }}, identifier
 	var output []*{{ .TagPackage }}.{{ or .TagType2 .TagType }}
     {{- end }}
 
+	{{ if .RetryTagsListTagsType }}
+	_, err := tfresource.RetryWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+		func() (string, error) {
+			return "", conn.{{ .ListTagsOp }}PagesWithContext(ctx, input, func(page *{{ .TagPackage  }}.{{ .ListTagsOp }}Output, lastPage bool) bool {
+				if page == nil {
+					return !lastPage
+				}
+
+			{{ if .ServiceTagsMap }}
+				maps.Copy(output, page.{{ .ListTagsOutTagsElem }})
+			{{- else }}
+				for _, v := range page.{{ .ListTagsOutTagsElem }} {
+					if v != nil {
+						output = append(output, v)
+					}
+				}
+			{{- end }}
+
+				return !lastPage
+			})
+		},
+		[]string{
+			{{- range .RetryTagsErrorCodes }}
+			{{ . }},
+			{{- end }}
+		},
+		[]string{
+			{{- range .RetryTagsErrorMessages }}
+			"{{ . }}",
+			{{- end }}
+		},
+	)
+	{{ else }}
 	err := conn.{{ .ListTagsOp }}PagesWithContext(ctx, input, func(page *{{ .TagPackage  }}.{{ .ListTagsOp }}Output, lastPage bool) bool {
 		if page == nil {
 			return !lastPage
@@ -45,9 +78,29 @@ func {{ .ListTagsFunc }}(ctx context.Context, conn {{ .ClientType }}, identifier
 
 		return !lastPage
 	})
+	{{- end }}
 {{ else }}
+	{{- if .RetryTagsListTagsType }}
+
+	output, err := tfresource.RetryGWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+		func() (*{{ .TagPackage  }}.{{ .RetryTagsListTagsType }}, error) {
+			return conn.{{ .ListTagsOp }}WithContext(ctx, input)
+		},
+		[]string{
+			{{- range .RetryTagsErrorCodes }}
+			{{ . }},
+			{{- end }}
+		},
+		[]string{
+			{{- range .RetryTagsErrorMessages }}
+			"{{ . }}",
+			{{- end }}
+		},
+	)
+	{{- else }}
 
 	output, err := conn.{{ .ListTagsOp }}WithContext(ctx, input)
+	{{- end }}
 {{- end }}
 
 	{{ if and ( .ParentNotFoundErrCode ) ( .ParentNotFoundErrMsg ) }}

internal/generate/tags/templates/v1/update_tags_body.tmpl

@@ -60,7 +60,25 @@ func {{ .UpdateTagsFunc }}(ctx context.Context, conn {{ .ClientType }}, identifi
 		{{- end }}
 	}
 
+	{{ if .RetryTagsListTagsType }}
+	_, err := tfresource.RetryWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+		func() (any, error) {
+			return conn.{{ .TagOp }}WithContext(ctx, input)
+		},
+		[]string{
+			{{- range .RetryTagsErrorCodes }}
+			{{ . }},
+			{{- end }}
+		},
+		[]string{
+			{{- range .RetryTagsErrorMessages }}
+			"{{ . }}",
+			{{- end }}
+		},
+	)
+	{{ else }}
 	_, err := conn.{{ .TagOp }}WithContext(ctx, input)
+	{{- end }}
 
 	if err != nil {
 		return fmt.Errorf("tagging resource (%s): %w", identifier, err)
@@ -100,7 +118,25 @@ func {{ .UpdateTagsFunc }}(ctx context.Context, conn {{ .ClientType }}, identifi
 			{{- end }}
 		}
 
+		{{ if .RetryTagsListTagsType }}
+		_, err := tfresource.RetryWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+			func() (any, error) {
+				return conn.{{ .UntagOp }}WithContext(ctx, input)
+			},
+			[]string{
+				{{- range .RetryTagsErrorCodes }}
+				{{ . }},
+				{{- end }}
+			},
+			[]string{
+				{{- range .RetryTagsErrorMessages }}
+				"{{ . }}",
+				{{- end }}
+			},
+		)
+		{{ else }}
 		_, err := conn.{{ .UntagOp }}WithContext(ctx, input)
+		{{- end }}
 
 		if err != nil {
 			return fmt.Errorf("untagging resource (%s): %w", identifier, err)
@@ -138,7 +174,25 @@ func {{ .UpdateTagsFunc }}(ctx context.Context, conn {{ .ClientType }}, identifi
 			{{- end }}
 		}
 
+		{{ if .RetryTagsListTagsType }}
+		_, err := tfresource.RetryWhenMessageContains(ctx, {{ .RetryTagsTimeout }},
+			func() (any, error) {
+				return conn.{{ .TagOp }}WithContext(ctx, input)
+			},
+			[]string{
+				{{- range .RetryTagsErrorCodes }}
+				{{ . }},
+				{{- end }}
+			},
+			[]string{
+				{{- range .RetryTagsErrorMessages }}
+				"{{ . }}",
+				{{- end }}
+			},
+		)
+		{{ else }}
 		_, err := conn.{{ .TagOp }}WithContext(ctx, input)
+		{{- end }}
 
 		if err != nil {
 			return fmt.Errorf("tagging resource (%s): %w", identifier, err)

internal/service/elasticache/generate.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
 
-//go:generate go run ../../generate/tags/main.go -ListTags -ListTagsInIDElem=ResourceName -ListTagsOutTagsElem=TagList -ServiceTagsSlice -TagOp=AddTagsToResource -TagInIDElem=ResourceName -UntagOp=RemoveTagsFromResource -UpdateTags -CreateTags
+//go:generate go run ../../generate/tags/main.go -ListTags -ListTagsInIDElem=ResourceName -ListTagsOutTagsElem=TagList -ServiceTagsSlice -TagOp=AddTagsToResource -TagInIDElem=ResourceName -UntagOp=RemoveTagsFromResource -UpdateTags -CreateTags -RetryTagsListTagsType=TagListMessage -RetryTagsErrorCodes=elasticache.ErrCodeInvalidReplicationGroupStateFault "-RetryTagsErrorMessages=not in available state" -RetryTagsTimeout=15m
 //go:generate go run ../../generate/tags/main.go -AWSSDKVersion=2 -TagsFunc=TagsV2 -KeyValueTagsFunc=keyValueTagsV2 -GetTagsInFunc=getTagsInV2 -SetTagsOutFunc=setTagsOutV2 -SkipAWSServiceImp -KVTValues -ServiceTagsSlice -- tagsv2_gen.go
 //go:generate go run ../../generate/servicepackage/main.go
 // ONLY generate directives and package declaration! Do not add anything else to this file.

internal/service/elasticache/replication_group.go

@@ -862,7 +862,13 @@ func resourceReplicationGroupUpdate(ctx context.Context, d *schema.ResourceData,
 		}
 
 		if requestUpdate {
-			_, err := conn.ModifyReplicationGroupWithContext(ctx, input)
+			// tagging may cause this resource to not yet be available, so wait for it to be available
+			_, err := WaitReplicationGroupAvailable(ctx, conn, d.Id(), d.Timeout(schema.TimeoutUpdate))
+			if err != nil {
+				return sdkdiag.AppendErrorf(diags, "waiting for ElastiCache Replication Group (%s) to update: %s", d.Id(), err)
+			}
+
+			_, err = conn.ModifyReplicationGroupWithContext(ctx, input)
 			if err != nil {
 				return sdkdiag.AppendErrorf(diags, "updating ElastiCache Replication Group (%s): %s", d.Id(), err)
 			}
@@ -881,7 +887,13 @@ func resourceReplicationGroupUpdate(ctx context.Context, d *schema.ResourceData,
 				AuthToken:               aws.String(d.Get("auth_token").(string)),
 			}
 
-			_, err := conn.ModifyReplicationGroupWithContext(ctx, params)
+			// tagging may cause this resource to not yet be available, so wait for it to be available
+			_, err := WaitReplicationGroupAvailable(ctx, conn, d.Id(), d.Timeout(schema.TimeoutUpdate))
+			if err != nil {
+				return sdkdiag.AppendErrorf(diags, "waiting for ElastiCache Replication Group (%s) to update: %s", d.Id(), err)
+			}
+
+			_, err = conn.ModifyReplicationGroupWithContext(ctx, params)
 			if err != nil {
 				return sdkdiag.AppendErrorf(diags, "changing auth_token for ElastiCache Replication Group (%s): %s", d.Id(), err)
 			}