Fix wait for rollbacked deployment with CodeDeploy.

appspec/spec.go

@@ -30,6 +30,14 @@ func (a *AppSpec) String() string {
 	return string(b)
 }
 
+func Unmarsal(data []byte) (*AppSpec, error) {
+	var spec AppSpec
+	if err := yaml.Unmarshal(data, &spec); err != nil {
+		return nil, err
+	}
+	return &spec, nil
+}
+
 func NewWithService(sv *types.Service, tdArn string) (*AppSpec, error) {
 	if len(sv.LoadBalancers) == 0 {
 		return nil, errors.New("require LoadBalancers")

appspec/spec_test.go

@@ -70,4 +70,13 @@ func TestAppSpec(t *testing.T) {
 	if diff := cmp.Diff(&s, expected); diff != "" {
 		t.Error(diff)
 	}
+
+	// round trip
+	r, err := appspec.Unmarsal([]byte(s.String()))
+	if err != nil {
+		t.Error(err)
+	}
+	if diff := cmp.Diff(r, expected); diff != "" {
+		t.Error("failed to Unmarsal", diff)
+	}
 }

ecspresso.go

@@ -59,6 +59,10 @@ func (d *App) newServiceFromTypes(ctx context.Context, in types.Service) (*Servi
 	dps := lo.Filter(in.Deployments, func(dp types.Deployment, i int) bool {
 		return aws.ToString(dp.Status) == "PRIMARY"
 	})
+	if sv.isCodeDeploy() {
+		// CodeDeploy does not support ServiceConnectConfiguration and VolumeConfigurations
+		return &sv, nil
+	}
 	if len(dps) == 0 {
 		d.Log("[WARNING] no primary deployment")
 		return &sv, nil

rollback.go

@@ -2,6 +2,7 @@ package ecspresso
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -11,6 +12,8 @@ import (
 	cdTypes "github.com/aws/aws-sdk-go-v2/service/codedeploy/types"
 	"github.com/aws/aws-sdk-go-v2/service/ecs"
 	"github.com/aws/aws-sdk-go-v2/service/ecs/types"
+	"github.com/kayac/ecspresso/v2/appspec"
+	"github.com/shogo82148/go-retry"
 )
 
 type RollbackOption struct {
@@ -41,125 +44,161 @@ func (d *App) Rollback(ctx context.Context, opt RollbackOption) error {
 		return err
 	}
 
-	currentArn := *sv.TaskDefinition
-	targetArn, err := d.FindRollbackTarget(ctx, currentArn)
-	if err != nil {
-		return err
-	}
-
-	d.Log("Rolling back to %s", arnToName(targetArn))
-	if opt.DryRun {
-		if opt.DeregisterTaskDefinition {
-			d.Log("%s will be deregistered", arnToName(currentArn))
-		} else {
-			d.Log("%s will not be deregistered", arnToName(currentArn))
-		}
-		d.Log("DRY RUN OK")
-		return nil
-	}
-
 	d.Log("deployment controller: %s", sv.DeploymentController.Type)
 	doRollback, err := d.RollbackFunc(sv)
 	if err != nil {
 		return err
 	}
+
 	doWait, err := d.WaitFunc(sv)
 	if err != nil {
 		return err
 	}
 
-	if err := doRollback(ctx, sv, targetArn, opt); err != nil {
+	// doRollback returns the task definition arn to be rolled back
+	rollbackedTdArn, err := doRollback(ctx, sv, opt)
+	if err != nil {
 		return err
 	}
 
+	if opt.DryRun {
+		if err := d.rollbackTaskDefinition(ctx, rollbackedTdArn, opt); err != nil {
+			return err
+		}
+		d.Log("DRY RUN OK")
+		return nil
+	}
+
 	if !opt.Wait {
 		d.Log("Service is rolled back.")
 		return nil
 	}
 
 	time.Sleep(delayForServiceChanged) // wait for service updated
 	if err := doWait(ctx, sv); err != nil {
+		if errors.As(err, &errNotFound) {
+			d.Log("[INFO] %s", err)
+			return nil
+		}
 		return err
 	}
 
 	d.Log("Service is stable now. Completed!")
 
-	if opt.DeregisterTaskDefinition {
-		d.Log("Deregistering the rolled-back task definition %s", arnToName(currentArn))
-		_, err := d.ecs.DeregisterTaskDefinition(
-			ctx,
-			&ecs.DeregisterTaskDefinitionInput{
-				TaskDefinition: &currentArn,
-			},
-		)
-		if err != nil {
-			return fmt.Errorf("failed to deregister task definition: %w", err)
-		}
-		d.Log("%s was deregistered successfully", arnToName(currentArn))
+	return d.rollbackTaskDefinition(ctx, rollbackedTdArn, opt)
+}
+
+func (d *App) rollbackTaskDefinition(ctx context.Context, rollbackedTdArn string, opt RollbackOption) error {
+	if !opt.DeregisterTaskDefinition {
+		return nil
+	}
+	if opt.DryRun {
+		d.Log("task definition %s will be deregistered", arnToName(rollbackedTdArn))
+		return nil
 	}
 
+	d.Log("Deregistering the rolled-back task definition %s", arnToName(rollbackedTdArn))
+	_, err := d.ecs.DeregisterTaskDefinition(
+		ctx,
+		&ecs.DeregisterTaskDefinitionInput{
+			TaskDefinition: &rollbackedTdArn,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to deregister task definition: %w", err)
+	}
+	d.Log("%s was deregistered successfully", arnToName(rollbackedTdArn))
 	return nil
 }
 
-func (d *App) RollbackServiceTasks(ctx context.Context, sv *Service, tdArn string, opt RollbackOption) error {
-	return d.UpdateServiceTasks(
+func (d *App) RollbackServiceTasks(ctx context.Context, sv *Service, opt RollbackOption) (string, error) {
+	currentArn := *sv.TaskDefinition
+	targetArn, err := d.FindRollbackTarget(ctx, currentArn)
+	if err != nil {
+		return "", err
+	}
+
+	d.Log("Rolling back to %s %s", arnToName(targetArn), opt.DryRunString())
+	if opt.DryRun {
+		return currentArn, nil
+	}
+
+	if err := d.UpdateServiceTasks(
 		ctx,
-		tdArn,
+		targetArn,
 		nil,
 		sv,
 		DeployOption{
 			ForceNewDeployment: false,
 			UpdateService:      false,
 		},
-	)
+	); err != nil {
+		return "", err
+	}
+	return currentArn, nil
 }
 
-func (d *App) RollbackByCodeDeploy(ctx context.Context, sv *Service, tdArn string, opt RollbackOption) error {
+func (d *App) RollbackByCodeDeploy(ctx context.Context, sv *Service, opt RollbackOption) (string, error) {
 	dp, err := d.findDeploymentInfo(ctx)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	ld, err := d.codedeploy.ListDeployments(ctx, &codedeploy.ListDeploymentsInput{
 		ApplicationName:     dp.ApplicationName,
 		DeploymentGroupName: dp.DeploymentGroupName,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to list deployments: %w", err)
+		return "", fmt.Errorf("failed to list deployments: %w", err)
 	}
 	if len(ld.Deployments) == 0 {
-		return ErrNotFound("no deployments are found")
+		return "", ErrNotFound("no deployments are found")
 	}
 
-	dpID := ld.Deployments[0] // latest deployment id
-
-	dep, err := d.codedeploy.GetDeployment(ctx, &codedeploy.GetDeploymentInput{
-		DeploymentId: &dpID,
+	out, err := d.codedeploy.GetDeployment(ctx, &codedeploy.GetDeploymentInput{
+		DeploymentId: &ld.Deployments[0], // latest deployment
 	})
 	if err != nil {
-		return fmt.Errorf("failed to get deployment: %w", err)
+		return "", fmt.Errorf("failed to get deployment: %w", err)
 	}
+	currentDeployment := out.DeploymentInfo
 
-	if opt.DryRun {
-		d.Log("deployment id: %s", dpID)
-		d.Log("DRY RUN OK")
-		return nil
-	}
+	d.Log("current deployment id: %s", *currentDeployment.DeploymentId)
 
-	switch dep.DeploymentInfo.Status {
+	switch currentDeployment.Status {
 	case cdTypes.DeploymentStatusSucceeded, cdTypes.DeploymentStatusFailed, cdTypes.DeploymentStatusStopped:
-		return d.createDeployment(ctx, sv, tdArn, opt.RollbackEvents)
+		currentTdArn := *sv.TaskDefinition
+		targetArn, err := d.FindRollbackTarget(ctx, currentTdArn)
+		if err != nil {
+			return "", err
+		}
+		d.Log("the deployment in progress is not found, creating a new deployment with %s %s", targetArn, opt.DryRunString())
+		if opt.DryRun {
+			return currentTdArn, nil
+		}
+		if err := d.createDeployment(ctx, sv, targetArn, opt.RollbackEvents); err != nil {
+			return "", err
+		}
+		return currentTdArn, nil
 	default: // If the deployment is not yet complete
-		_, err = d.codedeploy.StopDeployment(ctx, &codedeploy.StopDeploymentInput{
-			DeploymentId:        &dpID,
-			AutoRollbackEnabled: aws.Bool(true),
-		})
+		d.Log("the deployment in progress found, stopping the deployment %s %s", *currentDeployment.DeploymentId, opt.DryRunString())
+		tdArn, err := d.findTaskDefinitionOfDeployment(ctx, currentDeployment)
 		if err != nil {
-			return fmt.Errorf("failed to roll back the deployment: %w", err)
+			return "", err
 		}
-
-		d.Log("Deployment %s is rolled back on CodeDeploy:", dpID)
-		return nil
+		if opt.DryRun {
+			return tdArn, nil
+		}
+		if _, err := d.codedeploy.StopDeployment(ctx, &codedeploy.StopDeploymentInput{
+			DeploymentId:        currentDeployment.DeploymentId,
+			AutoRollbackEnabled: aws.Bool(true),
+		}); err != nil {
+			return "", fmt.Errorf("failed to roll back the deployment: %w", err)
+		}
+		if err := d.waitForDeploymentRollbacked(ctx, *currentDeployment.DeploymentId); err != nil {
+			return "", err
+		}
+		return tdArn, nil
 	}
 }
 
@@ -198,7 +237,7 @@ func (d *App) FindRollbackTarget(ctx context.Context, taskDefinitionArn string)
 	return "", ErrNotFound("rollback target is not found")
 }
 
-type rollbackFunc func(ctx context.Context, sv *Service, taskDefinitionArn string, opt RollbackOption) error
+type rollbackFunc func(ctx context.Context, sv *Service, opt RollbackOption) (string, error)
 
 func (d *App) RollbackFunc(sv *Service) (rollbackFunc, error) {
 	defaultFunc := d.RollbackServiceTasks
@@ -217,3 +256,42 @@ func (d *App) RollbackFunc(sv *Service) (rollbackFunc, error) {
 	}
 	return defaultFunc, nil
 }
+
+func (d *App) findTaskDefinitionOfDeployment(ctx context.Context, dp *cdTypes.DeploymentInfo) (string, error) {
+	resRev, err := d.codedeploy.GetApplicationRevision(ctx, &codedeploy.GetApplicationRevisionInput{
+		ApplicationName: dp.ApplicationName,
+		Revision:        dp.Revision,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to get application revision: %w", err)
+	}
+	spec, err := appspec.Unmarsal([]byte(*resRev.Revision.AppSpecContent.Content))
+	if err != nil {
+		return "", fmt.Errorf("failed to unmarshal appspec: %w", err)
+	}
+	return *spec.Resources[0].TargetService.Properties.TaskDefinition, nil
+}
+
+func (d *App) waitForDeploymentRollbacked(ctx context.Context, id string) error {
+	p := retry.Policy{
+		MinDelay: time.Second,
+		MaxDelay: 10 * time.Second,
+		MaxCount: 10,
+	}
+	return p.Do(ctx, func() error {
+		out, err := d.codedeploy.GetDeployment(ctx, &codedeploy.GetDeploymentInput{
+			DeploymentId: aws.String(id),
+		})
+		if err != nil {
+			return fmt.Errorf("failed to get deployment: %w", err)
+		}
+		status := out.DeploymentInfo.Status
+		rbinfo := out.DeploymentInfo.RollbackInfo
+		if status == cdTypes.DeploymentStatusStopped && rbinfo != nil && rbinfo.RollbackDeploymentId != nil {
+			d.Log("Deployment %s is stopped", id)
+			d.Log("Rollback deployment created: %s", *rbinfo.RollbackDeploymentId)
+			return nil
+		}
+		return fmt.Errorf("deployment %s is not stopped yet", id)
+	})
+}