diff --git a/builtin/providers/aws/resource_aws_cloudformation_stack.go b/builtin/providers/aws/resource_aws_cloudformation_stack.go index 1b767ddce651..226b40cd50a0 100644 --- a/builtin/providers/aws/resource_aws_cloudformation_stack.go +++ b/builtin/providers/aws/resource_aws_cloudformation_stack.go @@ -145,29 +145,46 @@ func resourceAwsCloudFormationStackCreate(d *schema.ResourceData, meta interface } d.SetId(*resp.StackId) + var lastStatus string wait := resource.StateChangeConf{ - Pending: []string{"CREATE_IN_PROGRESS", "ROLLBACK_IN_PROGRESS", "ROLLBACK_COMPLETE"}, - Target: []string{"CREATE_COMPLETE"}, + Pending: []string{ + "CREATE_IN_PROGRESS", + "DELETE_IN_PROGRESS", + "ROLLBACK_IN_PROGRESS", + }, + Target: []string{ + "CREATE_COMPLETE", + "CREATE_FAILED", + "DELETE_COMPLETE", + "DELETE_FAILED", + "ROLLBACK_COMPLETE", + "ROLLBACK_FAILED", + }, Timeout: time.Duration(retryTimeout) * time.Minute, - MinTimeout: 5 * time.Second, + MinTimeout: 1 * time.Second, Refresh: func() (interface{}, string, error) { resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: aws.String(d.Get("name").(string)), + StackName: aws.String(d.Id()), }) + if len(resp.Stacks) == 0 { + // This shouldn't happen unless CloudFormation is inconsistent + // See https://github.com/hashicorp/terraform/issues/5487 + log.Printf("[WARN] CloudFormation stack %q not found.\nresponse: %q", + d.Id(), resp) + return resp, "", fmt.Errorf( + "CloudFormation stack %q vanished unexpectedly during creation.\n"+ + "Unless you knowingly manually deleted the stack "+ + "please report this as bug at https://github.com/hashicorp/terraform/issues\n"+ + "along with the config & Terraform version & the details below:\n"+ + "Full API response: %s\n", + d.Id(), resp) + } + status := *resp.Stacks[0].StackStatus + lastStatus = status log.Printf("[DEBUG] Current CloudFormation stack status: %q", status) - if status == "ROLLBACK_COMPLETE" { - stack := resp.Stacks[0] - failures, err := getCloudFormationFailures(stack.StackName, *stack.CreationTime, conn) - if err != nil { - return resp, "", fmt.Errorf( - "Failed getting details about rollback: %q", err.Error()) - } - - return resp, "", fmt.Errorf("ROLLBACK_COMPLETE:\n%q", failures) - } return resp, status, err }, } @@ -177,17 +194,41 @@ func resourceAwsCloudFormationStackCreate(d *schema.ResourceData, meta interface return err } - log.Printf("[INFO] CloudFormation Stack %q created", d.Get("name").(string)) + if lastStatus == "ROLLBACK_COMPLETE" || lastStatus == "ROLLBACK_FAILED" { + reasons, err := getCloudFormationRollbackReasons(d.Id(), nil, conn) + if err != nil { + return fmt.Errorf("Failed getting rollback reasons: %q", err.Error()) + } + + return fmt.Errorf("%s: %q", lastStatus, reasons) + } + if lastStatus == "DELETE_COMPLETE" || lastStatus == "DELETE_FAILED" { + reasons, err := getCloudFormationDeletionReasons(d.Id(), conn) + if err != nil { + return fmt.Errorf("Failed getting deletion reasons: %q", err.Error()) + } + + d.SetId("") + return fmt.Errorf("%s: %q", lastStatus, reasons) + } + if lastStatus == "CREATE_FAILED" { + reasons, err := getCloudFormationFailures(d.Id(), conn) + if err != nil { + return fmt.Errorf("Failed getting failure reasons: %q", err.Error()) + } + return fmt.Errorf("%s: %q", lastStatus, reasons) + } + + log.Printf("[INFO] CloudFormation Stack %q created", d.Id()) return resourceAwsCloudFormationStackRead(d, meta) } func resourceAwsCloudFormationStackRead(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).cfconn - stackName := d.Get("name").(string) input := &cloudformation.DescribeStacksInput{ - StackName: aws.String(stackName), + StackName: aws.String(d.Id()), } resp, err := conn.DescribeStacks(input) if err != nil { @@ -218,7 +259,7 @@ func resourceAwsCloudFormationStackRead(d *schema.ResourceData, meta interface{} } tInput := cloudformation.GetTemplateInput{ - StackName: aws.String(stackName), + StackName: aws.String(d.Id()), } out, err := conn.GetTemplate(&tInput) if err != nil { @@ -280,7 +321,7 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface conn := meta.(*AWSClient).cfconn input := &cloudformation.UpdateStackInput{ - StackName: aws.String(d.Get("name").(string)), + StackName: aws.String(d.Id()), } // Either TemplateBody, TemplateURL or UsePreviousTemplate are required @@ -318,7 +359,7 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface return err } - lastUpdatedTime, err := getLastCfEventTimestamp(d.Get("name").(string), conn) + lastUpdatedTime, err := getLastCfEventTimestamp(d.Id(), conn) if err != nil { return err } @@ -330,36 +371,29 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface log.Printf("[DEBUG] CloudFormation timeout: %d", retryTimeout) } } + var lastStatus string wait := resource.StateChangeConf{ Pending: []string{ "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", "UPDATE_IN_PROGRESS", "UPDATE_ROLLBACK_IN_PROGRESS", "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", + }, + Target: []string{ + "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE", + "UPDATE_ROLLBACK_FAILED", }, - Target: []string{"UPDATE_COMPLETE"}, Timeout: time.Duration(retryTimeout) * time.Minute, MinTimeout: 5 * time.Second, Refresh: func() (interface{}, string, error) { resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: aws.String(d.Get("name").(string)), + StackName: aws.String(d.Id()), }) - stack := resp.Stacks[0] - status := *stack.StackStatus + status := *resp.Stacks[0].StackStatus + lastStatus = status log.Printf("[DEBUG] Current CloudFormation stack status: %q", status) - if status == "UPDATE_ROLLBACK_COMPLETE" { - failures, err := getCloudFormationFailures(stack.StackName, *lastUpdatedTime, conn) - if err != nil { - return resp, "", fmt.Errorf( - "Failed getting details about rollback: %q", err.Error()) - } - - return resp, "", fmt.Errorf( - "UPDATE_ROLLBACK_COMPLETE:\n%q", failures) - } - return resp, status, err }, } @@ -369,6 +403,15 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface return err } + if lastStatus == "UPDATE_ROLLBACK_COMPLETE" || lastStatus == "UPDATE_ROLLBACK_FAILED" { + reasons, err := getCloudFormationRollbackReasons(*stack.StackId, lastUpdatedTime, conn) + if err != nil { + return fmt.Errorf("Failed getting details about rollback: %q", err.Error()) + } + + return fmt.Errorf("%s: %q", lastStatus, reasons) + } + log.Printf("[DEBUG] CloudFormation stack %q has been updated", *stack.StackId) return resourceAwsCloudFormationStackRead(d, meta) @@ -378,7 +421,7 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface conn := meta.(*AWSClient).cfconn input := &cloudformation.DeleteStackInput{ - StackName: aws.String(d.Get("name").(string)), + StackName: aws.String(d.Id()), } log.Printf("[DEBUG] Deleting CloudFormation stack %s", input) _, err := conn.DeleteStack(input) @@ -394,21 +437,27 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface } return err } - + var lastStatus string wait := resource.StateChangeConf{ - Pending: []string{"DELETE_IN_PROGRESS", "ROLLBACK_IN_PROGRESS"}, - Target: []string{"DELETE_COMPLETE"}, + Pending: []string{ + "DELETE_IN_PROGRESS", + "ROLLBACK_IN_PROGRESS", + }, + Target: []string{ + "DELETE_COMPLETE", + "DELETE_FAILED", + "__ERROR", + }, Timeout: 30 * time.Minute, MinTimeout: 5 * time.Second, Refresh: func() (interface{}, string, error) { resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: aws.String(d.Get("name").(string)), + StackName: aws.String(d.Id()), }) - if err != nil { awsErr, ok := err.(awserr.Error) if !ok { - return resp, "DELETE_FAILED", err + return resp, "__ERROR", err } log.Printf("[DEBUG] Error when deleting CloudFormation stack: %s: %s", @@ -421,11 +470,12 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface } if len(resp.Stacks) == 0 { - log.Printf("[DEBUG] CloudFormation stack %q is already gone", d.Get("name")) + log.Printf("[DEBUG] CloudFormation stack %q is already gone", d.Id()) return resp, "DELETE_COMPLETE", nil } status := *resp.Stacks[0].StackStatus + lastStatus = status log.Printf("[DEBUG] Current CloudFormation stack status: %q", status) return resp, status, err @@ -437,6 +487,15 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface return err } + if lastStatus == "DELETE_FAILED" { + reasons, err := getCloudFormationFailures(d.Id(), conn) + if err != nil { + return fmt.Errorf("Failed getting reasons of failure: %q", err.Error()) + } + + return fmt.Errorf("%s: %q", lastStatus, reasons) + } + log.Printf("[DEBUG] CloudFormation stack %q has been deleted", d.Id()) d.SetId("") @@ -460,32 +519,73 @@ func getLastCfEventTimestamp(stackName string, conn *cloudformation.CloudFormati return output.StackEvents[0].Timestamp, nil } -// getCloudFormationFailures returns ResourceStatusReason(s) -// of events that should be failures based on regexp match of status -func getCloudFormationFailures(stackName *string, afterTime time.Time, - conn *cloudformation.CloudFormation) ([]string, error) { +func getCloudFormationRollbackReasons(stackId string, afterTime *time.Time, conn *cloudformation.CloudFormation) ([]string, error) { var failures []string - // Only catching failures from last 100 events - // Some extra iteration logic via NextToken could be added - // but in reality it's nearly impossible to generate >100 - // events by a single stack update - events, err := conn.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{ - StackName: stackName, + + err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{ + StackName: aws.String(stackId), + }, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool { + for _, e := range page.StackEvents { + if afterTime != nil && !e.Timestamp.After(*afterTime) { + continue + } + + if cfStackEventIsFailure(e) || cfStackEventIsRollback(e) { + failures = append(failures, *e.ResourceStatusReason) + } + } + return !lastPage }) - if err != nil { - return nil, err - } + return failures, err +} - failRe := regexp.MustCompile("_FAILED$") - rollbackRe := regexp.MustCompile("^ROLLBACK_") +func getCloudFormationDeletionReasons(stackId string, conn *cloudformation.CloudFormation) ([]string, error) { + var failures []string - for _, e := range events.StackEvents { - if (failRe.MatchString(*e.ResourceStatus) || rollbackRe.MatchString(*e.ResourceStatus)) && - e.Timestamp.After(afterTime) && e.ResourceStatusReason != nil { - failures = append(failures, *e.ResourceStatusReason) + err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{ + StackName: aws.String(stackId), + }, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool { + for _, e := range page.StackEvents { + if cfStackEventIsFailure(e) || cfStackEventIsStackDeletion(e) { + failures = append(failures, *e.ResourceStatusReason) + } } - } + return !lastPage + }) + + return failures, err +} + +func getCloudFormationFailures(stackId string, conn *cloudformation.CloudFormation) ([]string, error) { + var failures []string + + err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{ + StackName: aws.String(stackId), + }, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool { + for _, e := range page.StackEvents { + if cfStackEventIsFailure(e) { + failures = append(failures, *e.ResourceStatusReason) + } + } + return !lastPage + }) + + return failures, err +} + +func cfStackEventIsFailure(event *cloudformation.StackEvent) bool { + failRe := regexp.MustCompile("_FAILED$") + return failRe.MatchString(*event.ResourceStatus) && event.ResourceStatusReason != nil +} + +func cfStackEventIsRollback(event *cloudformation.StackEvent) bool { + rollbackRe := regexp.MustCompile("^ROLLBACK_") + return rollbackRe.MatchString(*event.ResourceStatus) && event.ResourceStatusReason != nil +} - return failures, nil +func cfStackEventIsStackDeletion(event *cloudformation.StackEvent) bool { + return *event.ResourceStatus == "DELETE_IN_PROGRESS" && + *event.ResourceType == "AWS::CloudFormation::Stack" && + event.ResourceStatusReason != nil }