Skip to content

Commit

Permalink
provider/aws: Cover more failure cases of CF creation/update/deletion
Browse files Browse the repository at this point in the history 
 - Use ID instead of stack name per AWS' advice
 - Query failures/errors *after* the stack has reached target state
  • Loading branch information
@radeksimko
radeksimko committed Aug 29, 2016
1 parent fe0e8da commit 4e90fc6
Showing 1 changed file with 164 additions and 64 deletions.
228 changes: 164 additions & 64 deletions builtin/providers/aws/resource_aws_cloudformation_stack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,29 +145,46 @@ func resourceAwsCloudFormationStackCreate(d *schema.ResourceData, meta interface
}

d.SetId(*resp.StackId)
var lastStatus string

wait := resource.StateChangeConf{
Pending: []string{"CREATE_IN_PROGRESS", "ROLLBACK_IN_PROGRESS", "ROLLBACK_COMPLETE"},
Target: []string{"CREATE_COMPLETE"},
Pending: []string{
"CREATE_IN_PROGRESS",
"DELETE_IN_PROGRESS",
"ROLLBACK_IN_PROGRESS",
},
Target: []string{
"CREATE_COMPLETE",
"CREATE_FAILED",
"DELETE_COMPLETE",
"DELETE_FAILED",
"ROLLBACK_COMPLETE",
"ROLLBACK_FAILED",
},
Timeout: time.Duration(retryTimeout) * time.Minute,
MinTimeout: 5 * time.Second,
MinTimeout: 1 * time.Second,
Refresh: func() (interface{}, string, error) {
resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(d.Get("name").(string)),
StackName: aws.String(d.Id()),
})
if len(resp.Stacks) == 0 {
// This shouldn't happen unless CloudFormation is inconsistent
// See https://github.com/hashicorp/terraform/issues/5487
log.Printf("[WARN] CloudFormation stack %q not found.\nresponse: %q",
d.Id(), resp)
return resp, "", fmt.Errorf(
"CloudFormation stack %q vanished unexpectedly during creation.\n"+
"Unless you knowingly manually deleted the stack "+
"please report this as bug at https://github.com/hashicorp/terraform/issues\n"+
"along with the config & Terraform version & the details below:\n"+
"Full API response: %s\n",
d.Id(), resp)
}

status := *resp.Stacks[0].StackStatus
lastStatus = status
log.Printf("[DEBUG] Current CloudFormation stack status: %q", status)

if status == "ROLLBACK_COMPLETE" {
stack := resp.Stacks[0]
failures, err := getCloudFormationFailures(stack.StackName, *stack.CreationTime, conn)
if err != nil {
return resp, "", fmt.Errorf(
"Failed getting details about rollback: %q", err.Error())
}

return resp, "", fmt.Errorf("ROLLBACK_COMPLETE:\n%q", failures)
}
return resp, status, err
},
}
Expand All @@ -177,17 +194,41 @@ func resourceAwsCloudFormationStackCreate(d *schema.ResourceData, meta interface
return err
}

log.Printf("[INFO] CloudFormation Stack %q created", d.Get("name").(string))
if lastStatus == "ROLLBACK_COMPLETE" || lastStatus == "ROLLBACK_FAILED" {
reasons, err := getCloudFormationRollbackReasons(d.Id(), nil, conn)
if err != nil {
return fmt.Errorf("Failed getting rollback reasons: %q", err.Error())
}

return fmt.Errorf("%s: %q", lastStatus, reasons)
}
if lastStatus == "DELETE_COMPLETE" || lastStatus == "DELETE_FAILED" {
reasons, err := getCloudFormationDeletionReasons(d.Id(), conn)
if err != nil {
return fmt.Errorf("Failed getting deletion reasons: %q", err.Error())
}

d.SetId("")
return fmt.Errorf("%s: %q", lastStatus, reasons)
}
if lastStatus == "CREATE_FAILED" {
reasons, err := getCloudFormationFailures(d.Id(), conn)
if err != nil {
return fmt.Errorf("Failed getting failure reasons: %q", err.Error())
}
return fmt.Errorf("%s: %q", lastStatus, reasons)
}

log.Printf("[INFO] CloudFormation Stack %q created", d.Id())

return resourceAwsCloudFormationStackRead(d, meta)
}

func resourceAwsCloudFormationStackRead(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).cfconn
stackName := d.Get("name").(string)

input := &cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
StackName: aws.String(d.Id()),
}
resp, err := conn.DescribeStacks(input)
if err != nil {
Expand Down Expand Up @@ -218,7 +259,7 @@ func resourceAwsCloudFormationStackRead(d *schema.ResourceData, meta interface{}
}

tInput := cloudformation.GetTemplateInput{
StackName: aws.String(stackName),
StackName: aws.String(d.Id()),
}
out, err := conn.GetTemplate(&tInput)
if err != nil {
Expand Down Expand Up @@ -280,7 +321,7 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface
conn := meta.(*AWSClient).cfconn

input := &cloudformation.UpdateStackInput{
StackName: aws.String(d.Get("name").(string)),
StackName: aws.String(d.Id()),
}

// Either TemplateBody, TemplateURL or UsePreviousTemplate are required
Expand Down Expand Up @@ -318,7 +359,7 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface
return err
}

lastUpdatedTime, err := getLastCfEventTimestamp(d.Get("name").(string), conn)
lastUpdatedTime, err := getLastCfEventTimestamp(d.Id(), conn)
if err != nil {
return err
}
Expand All @@ -330,36 +371,29 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface
log.Printf("[DEBUG] CloudFormation timeout: %d", retryTimeout)
}
}
var lastStatus string
wait := resource.StateChangeConf{
Pending: []string{
"UPDATE_COMPLETE_CLEANUP_IN_PROGRESS",
"UPDATE_IN_PROGRESS",
"UPDATE_ROLLBACK_IN_PROGRESS",
"UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS",
},
Target: []string{
"UPDATE_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE",
"UPDATE_ROLLBACK_FAILED",
},
Target: []string{"UPDATE_COMPLETE"},
Timeout: time.Duration(retryTimeout) * time.Minute,
MinTimeout: 5 * time.Second,
Refresh: func() (interface{}, string, error) {
resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(d.Get("name").(string)),
StackName: aws.String(d.Id()),
})
stack := resp.Stacks[0]
status := *stack.StackStatus
status := *resp.Stacks[0].StackStatus
lastStatus = status
log.Printf("[DEBUG] Current CloudFormation stack status: %q", status)

if status == "UPDATE_ROLLBACK_COMPLETE" {
failures, err := getCloudFormationFailures(stack.StackName, *lastUpdatedTime, conn)
if err != nil {
return resp, "", fmt.Errorf(
"Failed getting details about rollback: %q", err.Error())
}

return resp, "", fmt.Errorf(
"UPDATE_ROLLBACK_COMPLETE:\n%q", failures)
}

return resp, status, err
},
}
Expand All @@ -369,6 +403,15 @@ func resourceAwsCloudFormationStackUpdate(d *schema.ResourceData, meta interface
return err
}

if lastStatus == "UPDATE_ROLLBACK_COMPLETE" || lastStatus == "UPDATE_ROLLBACK_FAILED" {
reasons, err := getCloudFormationRollbackReasons(*stack.StackId, lastUpdatedTime, conn)
if err != nil {
return fmt.Errorf("Failed getting details about rollback: %q", err.Error())
}

return fmt.Errorf("%s: %q", lastStatus, reasons)
}

log.Printf("[DEBUG] CloudFormation stack %q has been updated", *stack.StackId)

return resourceAwsCloudFormationStackRead(d, meta)
Expand All @@ -378,7 +421,7 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface
conn := meta.(*AWSClient).cfconn

input := &cloudformation.DeleteStackInput{
StackName: aws.String(d.Get("name").(string)),
StackName: aws.String(d.Id()),
}
log.Printf("[DEBUG] Deleting CloudFormation stack %s", input)
_, err := conn.DeleteStack(input)
Expand All @@ -394,21 +437,27 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface
}
return err
}

var lastStatus string
wait := resource.StateChangeConf{
Pending: []string{"DELETE_IN_PROGRESS", "ROLLBACK_IN_PROGRESS"},
Target: []string{"DELETE_COMPLETE"},
Pending: []string{
"DELETE_IN_PROGRESS",
"ROLLBACK_IN_PROGRESS",
},
Target: []string{
"DELETE_COMPLETE",
"DELETE_FAILED",
"__ERROR",
},
Timeout: 30 * time.Minute,
MinTimeout: 5 * time.Second,
Refresh: func() (interface{}, string, error) {
resp, err := conn.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(d.Get("name").(string)),
StackName: aws.String(d.Id()),
})

if err != nil {
awsErr, ok := err.(awserr.Error)
if !ok {
return resp, "DELETE_FAILED", err
return resp, "__ERROR", err
}

log.Printf("[DEBUG] Error when deleting CloudFormation stack: %s: %s",
Expand All @@ -421,11 +470,12 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface
}

if len(resp.Stacks) == 0 {
log.Printf("[DEBUG] CloudFormation stack %q is already gone", d.Get("name"))
log.Printf("[DEBUG] CloudFormation stack %q is already gone", d.Id())
return resp, "DELETE_COMPLETE", nil
}

status := *resp.Stacks[0].StackStatus
lastStatus = status
log.Printf("[DEBUG] Current CloudFormation stack status: %q", status)

return resp, status, err
Expand All @@ -437,6 +487,15 @@ func resourceAwsCloudFormationStackDelete(d *schema.ResourceData, meta interface
return err
}

if lastStatus == "DELETE_FAILED" {
reasons, err := getCloudFormationFailures(d.Id(), conn)
if err != nil {
return fmt.Errorf("Failed getting reasons of failure: %q", err.Error())
}

return fmt.Errorf("%s: %q", lastStatus, reasons)
}

log.Printf("[DEBUG] CloudFormation stack %q has been deleted", d.Id())

d.SetId("")
Expand All @@ -460,32 +519,73 @@ func getLastCfEventTimestamp(stackName string, conn *cloudformation.CloudFormati
return output.StackEvents[0].Timestamp, nil
}

// getCloudFormationFailures returns ResourceStatusReason(s)
// of events that should be failures based on regexp match of status
func getCloudFormationFailures(stackName *string, afterTime time.Time,
conn *cloudformation.CloudFormation) ([]string, error) {
func getCloudFormationRollbackReasons(stackId string, afterTime *time.Time, conn *cloudformation.CloudFormation) ([]string, error) {
var failures []string
// Only catching failures from last 100 events
// Some extra iteration logic via NextToken could be added
// but in reality it's nearly impossible to generate >100
// events by a single stack update
events, err := conn.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: stackName,

err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
StackName: aws.String(stackId),
}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
for _, e := range page.StackEvents {
if afterTime != nil && !e.Timestamp.After(*afterTime) {
continue
}

if cfStackEventIsFailure(e) || cfStackEventIsRollback(e) {
failures = append(failures, *e.ResourceStatusReason)
}
}
return !lastPage
})

if err != nil {
return nil, err
}
return failures, err
}

failRe := regexp.MustCompile("_FAILED$")
rollbackRe := regexp.MustCompile("^ROLLBACK_")
func getCloudFormationDeletionReasons(stackId string, conn *cloudformation.CloudFormation) ([]string, error) {
var failures []string

for _, e := range events.StackEvents {
if (failRe.MatchString(*e.ResourceStatus) || rollbackRe.MatchString(*e.ResourceStatus)) &&
e.Timestamp.After(afterTime) && e.ResourceStatusReason != nil {
failures = append(failures, *e.ResourceStatusReason)
err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
StackName: aws.String(stackId),
}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
for _, e := range page.StackEvents {
if cfStackEventIsFailure(e) || cfStackEventIsStackDeletion(e) {
failures = append(failures, *e.ResourceStatusReason)
}
}
}
return !lastPage
})

return failures, err
}

func getCloudFormationFailures(stackId string, conn *cloudformation.CloudFormation) ([]string, error) {
var failures []string

err := conn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
StackName: aws.String(stackId),
}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
for _, e := range page.StackEvents {
if cfStackEventIsFailure(e) {
failures = append(failures, *e.ResourceStatusReason)
}
}
return !lastPage
})

return failures, err
}

func cfStackEventIsFailure(event *cloudformation.StackEvent) bool {
failRe := regexp.MustCompile("_FAILED$")
return failRe.MatchString(*event.ResourceStatus) && event.ResourceStatusReason != nil
}

func cfStackEventIsRollback(event *cloudformation.StackEvent) bool {
rollbackRe := regexp.MustCompile("^ROLLBACK_")
return rollbackRe.MatchString(*event.ResourceStatus) && event.ResourceStatusReason != nil
}

return failures, nil
func cfStackEventIsStackDeletion(event *cloudformation.StackEvent) bool {
return *event.ResourceStatus == "DELETE_IN_PROGRESS" &&
*event.ResourceType == "AWS::CloudFormation::Stack" &&
event.ResourceStatusReason != nil
}

0 comments on commit 4e90fc6

Please sign in to comment.