From fc4a26403b83eb2ed037879e33fcee92258a3eea Mon Sep 17 00:00:00 2001 From: Roberto Tyley Date: Fri, 31 May 2024 17:11:05 +0100 Subject: [PATCH] WIP: Add WaitForCullToComplete task --- .../magenta/deployment_type/AutoScaling.scala | 5 + .../main/scala/magenta/tasks/ASGTasks.scala | 182 ++++++++---------- .../src/main/scala/magenta/tasks/AWS.scala | 7 +- .../scala/magenta/tasks/ChangeSetTasks.scala | 6 +- .../tasks/autoscaling/CullSummary.scala | 67 +++++++ .../deployment_type/AutoScalingTest.scala | 146 ++++++-------- .../scala/magenta/tasks/ASGTasksTest.scala | 17 +- 7 files changed, 229 insertions(+), 201 deletions(-) create mode 100644 magenta-lib/src/main/scala/magenta/tasks/autoscaling/CullSummary.scala diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala b/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala index ca5bd71d4..28d1b0ceb 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala @@ -203,6 +203,11 @@ object AutoScaling extends DeploymentType with BucketParameters { terminationGrace(pkg, target, reporter) ), ResumeAlarmNotifications(autoScalingGroup, target.region), + WaitForCullToComplete( + autoScalingGroup, + secondsToWait(pkg, target, reporter), + target.region + ), WaitForStabilization( autoScalingGroup, secondsToWait(pkg, target, reporter), diff --git a/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala b/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala index 08061abbf..b4c34e523 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala @@ -1,11 +1,17 @@ package magenta.tasks import magenta.deployment_type.AutoScalingGroupInfo -import magenta.tasks.CullInstancesWithTerminationTag.TerminatingStates +import magenta.tasks.EC2.withEc2Client +import magenta.tasks.autoscaling.CullSummary import magenta.{KeyRing, _} import software.amazon.awssdk.awscore.exception.AwsServiceException import software.amazon.awssdk.services.autoscaling.AutoScalingClient -import software.amazon.awssdk.services.autoscaling.model.{AutoScalingGroup, Instance, LifecycleState, SetInstanceProtectionRequest} +import software.amazon.awssdk.services.autoscaling.model.{ + AutoScalingGroup, + Instance, + LifecycleState, + SetInstanceProtectionRequest +} import software.amazon.awssdk.services.ec2.Ec2Client import java.time.Duration @@ -17,9 +23,8 @@ case class CheckGroupSize(info: AutoScalingGroupInfo, region: Region)(implicit override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { val doubleCapacity = asg.desiredCapacity * 2 resources.reporter.verbose( s"ASG desired = ${asg.desiredCapacity}; ASG max = ${asg.maxSize}; Target = $doubleCapacity" @@ -43,11 +48,10 @@ case class TagCurrentInstancesWithTerminationTag( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { if (asg.instances.asScala.nonEmpty) { - EC2.withEc2Client(keyRing, region, resources) { ec2Client => + withEc2Client(keyRing, region, resources) { ec2Client => resources.reporter.verbose( s"Tagging ${asg.instances.asScala.toList.map(_.instanceId).mkString(", ")}" ) @@ -73,9 +77,8 @@ case class ProtectCurrentInstances(info: AutoScalingGroupInfo, region: Region)( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { val instances = asg.instances.asScala.toList val instancesInService = instances.filter(_.lifecycleState == LifecycleState.IN_SERVICE) @@ -104,9 +107,8 @@ case class DoubleSize(info: AutoScalingGroupInfo, region: Region)(implicit override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { val targetCapacity = asg.desiredCapacity * 2 resources.reporter.verbose(s"Doubling capacity to $targetCapacity") ASG.desiredCapacity(asg.autoScalingGroupName, targetCapacity, asgClient) @@ -125,9 +127,8 @@ sealed abstract class Pause(duration: Duration)(implicit val keyRing: KeyRing) def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { if (asg.desiredCapacity == 0 && asg.instances.isEmpty) resources.reporter.verbose( "Skipping pause as there are no instances and desired capacity is zero" @@ -175,9 +176,8 @@ case class WaitForStabilization( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { ELB.withClient(keyRing, region, resources) { elbClient => check(resources.reporter, stopFlag) { try { @@ -188,10 +188,9 @@ case class WaitForStabilization( case Right(()) => true } } catch { - case e: AwsServiceException if isRateExceeded(e) => { + case e: AwsServiceException if isRateExceeded(e) => resources.reporter.info(e.getMessage) false - } } } } @@ -205,16 +204,6 @@ case class WaitForStabilization( s"Check the desired number of hosts in both the ASG ($asgName) and ELB are up and that the number of hosts match" } -object CullInstancesWithTerminationTag { - // See https://docs.aws.amazon.com/autoscaling/ec2/userguide/lifecycle-hooks.html#lifecycle-hooks-overview - val TerminatingStates: Set[LifecycleState] = Set( - LifecycleState.TERMINATING, - LifecycleState.TERMINATING_WAIT, - LifecycleState.TERMINATING_PROCEED, - LifecycleState.TERMINATED - ) -} - case class CullInstancesWithTerminationTag( info: AutoScalingGroupInfo, region: Region @@ -223,49 +212,14 @@ case class CullInstancesWithTerminationTag( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { - EC2.withEc2Client(keyRing, region, resources) { ec2Client => - ELB.withClient(keyRing, region, resources) { elbClient => - val allInstances = asg.instances.asScala.toSeq - resources.reporter.verbose( - s"Found the following instances: ${allInstances.map(_.instanceId).mkString(", ")}" - ) - val instancesToKill = allInstances - .filter(instance => { - if ( - instance.lifecycleState == LifecycleState.UNKNOWN_TO_SDK_VERSION - ) { - logger.warn( - s"Instance lifecycle state ${instance.lifecycleStateAsString} isn't recognised in the AWS SDK. Is there a later version of the AWS SDK available?" - ) - } - - val isAlreadyTerminating = - TerminatingStates.contains(instance.lifecycleState) - val isTaggedForTermination = - EC2.hasTag(instance, "Magenta", "Terminate", ec2Client) - - isTaggedForTermination && !isAlreadyTerminating - }) - - reportOnRetainedInstances( - ec2Client, - resources, - allInstances, - instancesToKill - ) - - val orderedInstancesToKill = - instancesToKill.transposeBy(_.availabilityZone) + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { + withEc2Client(keyRing, region, resources) { implicit ec2Client => + val instancesToKill = + prepareOrderedInstancesToKill(asg, resources.reporter) + ELB.withClient(keyRing, region, resources) { implicit elbClient => try { - resources.reporter.verbose( - s"Culling instances: ${orderedInstancesToKill.map(_.instanceId).mkString(", ")}" - ) - orderedInstancesToKill.foreach(instance => - ASG.cull(asg, instance, asgClient, elbClient) - ) + instancesToKill.foreach(instance => ASG.cull(asg, instance)) } catch { case e: AwsServiceException if desiredSizeReset(e) => resources.reporter.warning( @@ -285,19 +239,33 @@ case class CullInstancesWithTerminationTag( .contains("ValidationError") } + private def prepareOrderedInstancesToKill( + asg: AutoScalingGroup, + reporter: DeployReporter + )(implicit ec2Client: Ec2Client): Seq[Instance] = { + val cullSummary = CullSummary.forAsg(asg, reporter) + reportOnRetainedInstances(reporter, cullSummary) + val orderedInstancesToKill = + cullSummary.instancesToKill.toSeq.transposeBy(_.availabilityZone) + reporter.verbose( + s"Culling instances: ${orderedInstancesToKill.map(_.instanceId).mkString(", ")}" + ) + orderedInstancesToKill + } + private def reportOnRetainedInstances( - ec2Client: Ec2Client, - resources: DeploymentResources, - allInstances: Seq[Instance], - instancesToKill: Seq[Instance] - ): Unit = { - val instancesToRetain = allInstances.diff(instancesToKill).toList - resources.reporter.verbose( + reporter: DeployReporter, + cullSummary: CullSummary + )(implicit ec2Client: Ec2Client): Unit = { + val instancesToKill = cullSummary.instancesToKill + val instancesToRetain = + cullSummary.allInstances.diff(instancesToKill).toList + reporter.verbose( s"Decided to keep the following instances: ${instancesToRetain.map(_.instanceId).mkString(", ")}" ) if (instancesToRetain.size != instancesToKill.size) { - resources.reporter.warning( + reporter.warning( s"Terminating ${instancesToKill.size} instances and retaining ${instancesToRetain.size} instances" ) logger.warn( @@ -306,10 +274,9 @@ case class CullInstancesWithTerminationTag( instancesToRetain.foreach(instanceToRetain => { val tags = EC2 .allTags(instanceToRetain, ec2Client) - .toList .map(tag => s"${tag.key}:${tag.value}") .mkString(", ") - resources.reporter.verbose( + reporter.verbose( s"Will not terminate $instanceToRetain. State: ${instanceToRetain.lifecycleStateAsString}. Tags: $tags" ) }) @@ -317,7 +284,30 @@ case class CullInstancesWithTerminationTag( } lazy val description = - s"Terminate instances in $asgName with the termination tag for this deploy" + s"Request termination for instances in $asgName with the termination tag for this deploy" +} + +case class WaitForCullToComplete( + info: AutoScalingGroupInfo, + duration: Duration, + region: Region +)(implicit val keyRing: KeyRing) + extends ASGTask + with SlowRepeatedPollingCheck { + + override def execute( + asg: AutoScalingGroup, + resources: DeploymentResources, + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = + withEc2Client(keyRing, region, resources) { implicit ec2Client => + check(resources.reporter, stopFlag) { + CullSummary.forAsg(asg, resources.reporter).isCullComplete + } + } + + lazy val description: String = + s"Check that all instances tagged for termination in $asgName have been terminated" } case class SuspendAlarmNotifications( @@ -329,9 +319,8 @@ case class SuspendAlarmNotifications( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = { ASG.suspendAlarmNotifications(asg.autoScalingGroupName, asgClient) } @@ -346,11 +335,9 @@ case class ResumeAlarmNotifications(info: AutoScalingGroupInfo, region: Region)( override def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit = { + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit = ASG.resumeAlarmNotifications(asg.autoScalingGroupName, asgClient) - } lazy val description = s"Resuming Alarm Notifications - $asgName will scale on any configured alarms" @@ -367,15 +354,14 @@ trait ASGTask extends Task { def execute( asg: AutoScalingGroup, resources: DeploymentResources, - stopFlag: => Boolean, - asgClient: AutoScalingClient - ): Unit + stopFlag: => Boolean + )(implicit asgClient: AutoScalingClient): Unit override def execute( resources: DeploymentResources, stopFlag: => Boolean ): Unit = { - ASG.withAsgClient(keyRing, region, resources) { asgClient => + ASG.withAsgClient(keyRing, region, resources) { implicit asgClient => resources.reporter.verbose( s"Looked up group matching tags ${info.tagRequirements}; identified ${info.asg.autoScalingGroupARN}" ) @@ -384,7 +370,7 @@ trait ASGTask extends Task { // For example, we need to make sure that we have the latest desired capacity when doubling the size of the ASG. val latestAsgState = ASG.getGroupByName(asgName, asgClient, resources.reporter) - execute(latestAsgState, resources, stopFlag, asgClient) + execute(latestAsgState, resources, stopFlag) } } } diff --git a/magenta-lib/src/main/scala/magenta/tasks/AWS.scala b/magenta-lib/src/main/scala/magenta/tasks/AWS.scala index 2d46f8fbc..a5173be79 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/AWS.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/AWS.scala @@ -403,9 +403,10 @@ object ASG { def cull( asg: AutoScalingGroup, - instance: ASGInstance, - asgClient: AutoScalingClient, - elbClient: ELB.Client + instance: ASGInstance + )(implicit + elbClient: ELB.Client, + asgClient: AutoScalingClient ): TerminateInstanceInAutoScalingGroupResponse = { ELB.deregister(elbNames(asg), elbTargetArns(asg), instance, elbClient) diff --git a/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala b/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala index cac05bfb5..9193f9abc 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala @@ -1,7 +1,11 @@ package magenta.tasks import magenta.artifact.S3Path -import magenta.tasks.CloudFormationParameters.{ExistingParameter, InputParameter, TemplateParameter} +import magenta.tasks.CloudFormationParameters.{ + ExistingParameter, + InputParameter, + TemplateParameter +} import magenta.tasks.UpdateCloudFormationTask._ import magenta.{DeployReporter, DeploymentResources, KeyRing, Region} import software.amazon.awssdk.services.cloudformation.CloudFormationClient diff --git a/magenta-lib/src/main/scala/magenta/tasks/autoscaling/CullSummary.scala b/magenta-lib/src/main/scala/magenta/tasks/autoscaling/CullSummary.scala new file mode 100644 index 000000000..033aef503 --- /dev/null +++ b/magenta-lib/src/main/scala/magenta/tasks/autoscaling/CullSummary.scala @@ -0,0 +1,67 @@ +package magenta.tasks.autoscaling + +import magenta.tasks.EC2 +import magenta.tasks.autoscaling.CullSummary.isNotAlreadyTerminating +import magenta.{DeployReporter, Loggable} +import software.amazon.awssdk.services.autoscaling.model.{ + AutoScalingGroup, + Instance, + LifecycleState +} +import software.amazon.awssdk.services.ec2.Ec2Client + +import scala.jdk.CollectionConverters._ + +object CullSummary extends Loggable { + // See https://docs.aws.amazon.com/autoscaling/ec2/userguide/lifecycle-hooks.html#lifecycle-hooks-overview + val TerminatingStates: Set[LifecycleState] = Set( + LifecycleState.TERMINATING, + LifecycleState.TERMINATING_WAIT, + LifecycleState.TERMINATING_PROCEED, + LifecycleState.TERMINATED + ) + + def isNotAlreadyTerminating(instance: Instance): Boolean = + !TerminatingStates(instance.lifecycleState) + + def logIfLifecycleStateUnknown(instance: Instance): Unit = if ( + instance.lifecycleState == LifecycleState.UNKNOWN_TO_SDK_VERSION + ) + logger.warn( + s"Instance lifecycle state ${instance.lifecycleStateAsString} isn't recognised in the AWS SDK. Is there a later version of the AWS SDK available?" + ) + + def isTaggedForTermination(instance: Instance)(implicit + ec2Client: Ec2Client + ): Boolean = + EC2.hasTag(instance, "Magenta", "Terminate", ec2Client) + + def forAsg(asg: AutoScalingGroup, reporter: DeployReporter)(implicit + ec2Client: Ec2Client + ): CullSummary = { + val allInstances = asg.instances.asScala.toSet + + reporter.verbose( + s"Found the following instances: ${allInstances.map(_.instanceId).mkString(", ")}" + ) + allInstances.foreach(logIfLifecycleStateUnknown) + + CullSummary( + allInstances, + instancesThatMustTerminate = allInstances.filter(isTaggedForTermination) + ) + } +} + +/** Summarises the instances in an ASG that should be culled - which ones are + * already terminating, and which ones must be told to terminate. + */ +case class CullSummary( + allInstances: Set[Instance], + instancesThatMustTerminate: Set[Instance] +) { + val instancesToKill: Set[Instance] = + instancesThatMustTerminate.filter(isNotAlreadyTerminating) + + val isCullComplete: Boolean = instancesThatMustTerminate.isEmpty +} diff --git a/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala b/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala index 4608d3b90..8a166810a 100644 --- a/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala +++ b/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala @@ -143,20 +143,21 @@ class AutoScalingTest DeployTarget(parameters(), stack, region) ) val expected = List( - WaitForStabilization(testInfo, ofMinutes(5), Region("eu-west-1")), - CheckGroupSize(testInfo, Region("eu-west-1")), - SuspendAlarmNotifications(testInfo, Region("eu-west-1")), - TagCurrentInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - ProtectCurrentInstances(testInfo, Region("eu-west-1")), - DoubleSize(testInfo, Region("eu-west-1")), - HealthcheckGrace(testInfo, Region("eu-west-1"), ofSeconds(20)), - WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")), - WarmupGrace(testInfo, Region("eu-west-1"), ofSeconds(1)), - WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")), - CullInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - TerminationGrace(testInfo, Region("eu-west-1"), ofSeconds(10)), - ResumeAlarmNotifications(testInfo, Region("eu-west-1")), - WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")) + WaitForStabilization(testInfo, ofMinutes(5), region), + CheckGroupSize(testInfo, region), + SuspendAlarmNotifications(testInfo, region), + TagCurrentInstancesWithTerminationTag(testInfo, region), + ProtectCurrentInstances(testInfo, region), + DoubleSize(testInfo, region), + HealthcheckGrace(testInfo, region, ofSeconds(20)), + WaitForStabilization(testInfo, ofMinutes(15), region), + WarmupGrace(testInfo, region, ofSeconds(1)), + WaitForStabilization(testInfo, ofMinutes(15), region), + CullInstancesWithTerminationTag(testInfo, region), + TerminationGrace(testInfo, region, ofSeconds(10)), + ResumeAlarmNotifications(testInfo, region), + WaitForCullToComplete(testInfo, ofMinutes(15), region), + WaitForStabilization(testInfo, ofMinutes(15), region) ) actual shouldBe expected } @@ -199,65 +200,37 @@ class AutoScalingTest ) val expected = List( // All tasks for testOldCfnAsg - WaitForStabilization(testOldCfnAsg, ofMinutes(5), Region("eu-west-1")), - CheckGroupSize(testOldCfnAsg, Region("eu-west-1")), - SuspendAlarmNotifications(testOldCfnAsg, Region("eu-west-1")), - TagCurrentInstancesWithTerminationTag( - testOldCfnAsg, - Region("eu-west-1") - ), - ProtectCurrentInstances(testOldCfnAsg, Region("eu-west-1")), - DoubleSize(testOldCfnAsg, Region("eu-west-1")), - HealthcheckGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(20)), - WaitForStabilization( - testOldCfnAsg, - ofMinutes(15), - Region("eu-west-1") - ), - WarmupGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(1)), - WaitForStabilization( - testOldCfnAsg, - ofMinutes(15), - Region("eu-west-1") - ), - CullInstancesWithTerminationTag(testOldCfnAsg, Region("eu-west-1")), - TerminationGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(10)), - ResumeAlarmNotifications(testOldCfnAsg, Region("eu-west-1")), - WaitForStabilization( - testOldCfnAsg, - ofMinutes(15), - Region("eu-west-1") - ), + WaitForStabilization(testOldCfnAsg, ofMinutes(5), region), + CheckGroupSize(testOldCfnAsg, region), + SuspendAlarmNotifications(testOldCfnAsg, region), + TagCurrentInstancesWithTerminationTag(testOldCfnAsg, region), + ProtectCurrentInstances(testOldCfnAsg, region), + DoubleSize(testOldCfnAsg, region), + HealthcheckGrace(testOldCfnAsg, region, ofSeconds(20)), + WaitForStabilization(testOldCfnAsg, ofMinutes(15), region), + WarmupGrace(testOldCfnAsg, region, ofSeconds(1)), + WaitForStabilization(testOldCfnAsg, ofMinutes(15), region), + CullInstancesWithTerminationTag(testOldCfnAsg, region), + TerminationGrace(testOldCfnAsg, region, ofSeconds(10)), + ResumeAlarmNotifications(testOldCfnAsg, region), + WaitForCullToComplete(testOldCfnAsg, ofMinutes(15), region), + WaitForStabilization(testOldCfnAsg, ofMinutes(15), region), // All tasks for testNewCdkAsg - WaitForStabilization(testNewCdkAsg, ofMinutes(5), Region("eu-west-1")), - CheckGroupSize(testNewCdkAsg, Region("eu-west-1")), - SuspendAlarmNotifications(testNewCdkAsg, Region("eu-west-1")), - TagCurrentInstancesWithTerminationTag( - testNewCdkAsg, - Region("eu-west-1") - ), - ProtectCurrentInstances(testNewCdkAsg, Region("eu-west-1")), - DoubleSize(testNewCdkAsg, Region("eu-west-1")), - HealthcheckGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(20)), - WaitForStabilization( - testNewCdkAsg, - ofMinutes(15), - Region("eu-west-1") - ), - WarmupGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(1)), - WaitForStabilization( - testNewCdkAsg, - ofMinutes(15), - Region("eu-west-1") - ), - CullInstancesWithTerminationTag(testNewCdkAsg, Region("eu-west-1")), - TerminationGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(10)), - ResumeAlarmNotifications(testNewCdkAsg, Region("eu-west-1")), - WaitForStabilization( - testNewCdkAsg, - ofMinutes(15), - Region("eu-west-1") - ) + WaitForStabilization(testNewCdkAsg, ofMinutes(5), region), + CheckGroupSize(testNewCdkAsg, region), + SuspendAlarmNotifications(testNewCdkAsg, region), + TagCurrentInstancesWithTerminationTag(testNewCdkAsg, region), + ProtectCurrentInstances(testNewCdkAsg, region), + DoubleSize(testNewCdkAsg, region), + HealthcheckGrace(testNewCdkAsg, region, ofSeconds(20)), + WaitForStabilization(testNewCdkAsg, ofMinutes(15), region), + WarmupGrace(testNewCdkAsg, region, ofSeconds(1)), + WaitForStabilization(testNewCdkAsg, ofMinutes(15), region), + CullInstancesWithTerminationTag(testNewCdkAsg, region), + TerminationGrace(testNewCdkAsg, region, ofSeconds(10)), + ResumeAlarmNotifications(testNewCdkAsg, region), + WaitForCullToComplete(testNewCdkAsg, ofMinutes(15), region), + WaitForStabilization(testNewCdkAsg, ofMinutes(15), region) ) actual shouldBe expected } @@ -335,20 +308,21 @@ class AutoScalingTest DeployTarget(parameters(), stack, region) ) val expected = List( - WaitForStabilization(testInfo, ofMinutes(5), Region("eu-west-1")), - CheckGroupSize(testInfo, Region("eu-west-1")), - SuspendAlarmNotifications(testInfo, Region("eu-west-1")), - TagCurrentInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - ProtectCurrentInstances(testInfo, Region("eu-west-1")), - DoubleSize(testInfo, Region("eu-west-1")), - HealthcheckGrace(testInfo, Region("eu-west-1"), ofSeconds(30)), - WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")), - WarmupGrace(testInfo, Region("eu-west-1"), ofSeconds(20)), - WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")), - CullInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - TerminationGrace(testInfo, Region("eu-west-1"), ofSeconds(11)), - ResumeAlarmNotifications(testInfo, Region("eu-west-1")), - WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")) + WaitForStabilization(testInfo, ofMinutes(5), region), + CheckGroupSize(testInfo, region), + SuspendAlarmNotifications(testInfo, region), + TagCurrentInstancesWithTerminationTag(testInfo, region), + ProtectCurrentInstances(testInfo, region), + DoubleSize(testInfo, region), + HealthcheckGrace(testInfo, region, ofSeconds(30)), + WaitForStabilization(testInfo, ofMinutes(3), region), + WarmupGrace(testInfo, region, ofSeconds(20)), + WaitForStabilization(testInfo, ofMinutes(3), region), + CullInstancesWithTerminationTag(testInfo, region), + TerminationGrace(testInfo, region, ofSeconds(11)), + ResumeAlarmNotifications(testInfo, region), + WaitForCullToComplete(testInfo, ofMinutes(3), region), + WaitForStabilization(testInfo, ofMinutes(3), region) ) actual shouldBe expected } diff --git a/magenta-lib/src/test/scala/magenta/tasks/ASGTasksTest.scala b/magenta-lib/src/test/scala/magenta/tasks/ASGTasksTest.scala index a6407f86e..e7c630a8c 100644 --- a/magenta-lib/src/test/scala/magenta/tasks/ASGTasksTest.scala +++ b/magenta-lib/src/test/scala/magenta/tasks/ASGTasksTest.scala @@ -41,7 +41,7 @@ class ASGTasksTest extends AnyFlatSpec with Matchers with MockitoSugar { .maxSize(10) .build() val info = AutoScalingGroupInfo(asg, Nil) - val asgClientMock = mock[AutoScalingClient] + implicit val asgClientMock = mock[AutoScalingClient] val p = DeploymentPackage( "test", @@ -60,7 +60,7 @@ class ASGTasksTest extends AnyFlatSpec with Matchers with MockitoSugar { mock[StsClient], global ) - task.execute(asg, resources, stopFlag = false, asgClientMock) + task.execute(asg, resources, stopFlag = false) verify(asgClientMock).setDesiredCapacity( SetDesiredCapacityRequest @@ -81,16 +81,7 @@ class ASGTasksTest extends AnyFlatSpec with Matchers with MockitoSugar { .build() val info = AutoScalingGroupInfo(asg, Nil) - val asgClientMock = mock[AutoScalingClient] - - val p = DeploymentPackage( - "test", - App("app"), - Map.empty, - "testDeploymentType", - S3Path("artifact-bucket", "project/123/test"), - deploymentTypes - ) + implicit val asgClientMock = mock[AutoScalingClient] val task = CheckGroupSize(info, Region("eu-west-1")) @@ -103,7 +94,7 @@ class ASGTasksTest extends AnyFlatSpec with Matchers with MockitoSugar { ) val thrown = intercept[FailException]( - task.execute(asg, resources, stopFlag = false, asgClientMock) + task.execute(asg, resources, stopFlag = false) ) thrown.getMessage should startWith(