From 0ccfcef8c8096dfebfca1a6f19a0e943daaf75c5 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Wed, 24 Jul 2024 08:38:39 +0100 Subject: [PATCH 01/23] fix(riff-raff.yaml): Do not deploy ASG w/update policy ASGs with an update policy will get deployed via CloudFormation, instead of Riff-Raff's `autoscaling` deployment type. --- src/riff-raff-yaml-file/index.test.ts | 79 +++++++++++++++++++++++++++ src/riff-raff-yaml-file/index.ts | 14 ++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/src/riff-raff-yaml-file/index.test.ts b/src/riff-raff-yaml-file/index.test.ts index 72ab74cc48..3a19a72f2f 100644 --- a/src/riff-raff-yaml-file/index.test.ts +++ b/src/riff-raff-yaml-file/index.test.ts @@ -1,4 +1,5 @@ import { App, Duration } from "aws-cdk-lib"; +import { UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; import { InstanceClass, InstanceSize, InstanceType } from "aws-cdk-lib/aws-ec2"; import { Schedule } from "aws-cdk-lib/aws-events"; import { Runtime } from "aws-cdk-lib/aws-lambda"; @@ -1255,4 +1256,82 @@ describe("The RiffRaffYamlFile class", () => { " `); }); + + it("Should only upload artifacts for an ASG with an update policy", () => { + const app = new App({ outdir: "/tmp/cdk.out" }); + + class MyApplicationStack extends GuStack { + // eslint-disable-next-line custom-rules/valid-constructors -- unit testing + constructor(app: App, id: string, props: GuStackProps) { + super(app, id, props); + + const appName = "my-app"; + + new GuEc2App(this, { + app: appName, + instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.MICRO), + access: { scope: AccessScope.PUBLIC }, + userData: { + distributable: { + fileName: `${appName}.deb`, + executionStatement: `dpkg -i /${appName}/${appName}.deb`, + }, + }, + certificateProps: { + domainName: "rip.gu.com", + }, + monitoringConfiguration: { noMonitoring: true }, + scaling: { + minimumInstances: 1, + }, + applicationPort: 9000, + imageRecipe: "arm64-bionic-java11-deploy-infrastructure", + updatePolicy: UpdatePolicy.rollingUpdate(), + }); + } + } + + new MyApplicationStack(app, "test-stack", { stack: "test", stage: "TEST", env: { region: "eu-west-1" } }); + + const actual = new RiffRaffYamlFile(app).toYAML(); + + expect(actual).toMatchInlineSnapshot(` + "allowedStages: + - TEST + deployments: + asg-upload-eu-west-1-test-my-app: + type: autoscaling + actions: + - uploadArtifacts + regions: + - eu-west-1 + stacks: + - test + app: my-app + parameters: + bucketSsmLookup: true + prefixApp: true + contentDirectory: my-app + cfn-eu-west-1-test-my-application-stack: + type: cloud-formation + regions: + - eu-west-1 + stacks: + - test + app: my-application-stack + contentDirectory: /tmp/cdk.out + parameters: + templateStagePaths: + TEST: test-stack.template.json + amiParametersToTags: + AMIMyapp: + BuiltBy: amigo + AmigoStage: PROD + Recipe: arm64-bionic-java11-deploy-infrastructure + Encrypted: 'true' + dependencies: + - asg-upload-eu-west-1-test-my-app + " + `); + }); }); diff --git a/src/riff-raff-yaml-file/index.ts b/src/riff-raff-yaml-file/index.ts index 71095d2a67..c33c453bb7 100644 --- a/src/riff-raff-yaml-file/index.ts +++ b/src/riff-raff-yaml-file/index.ts @@ -2,6 +2,7 @@ import { writeFileSync } from "fs"; import path from "path"; import type { App } from "aws-cdk-lib"; import { Token } from "aws-cdk-lib"; +import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; import { dump } from "js-yaml"; import { GuAutoScalingGroup } from "../constructs/autoscaling"; import { GuStack } from "../constructs/core"; @@ -253,7 +254,18 @@ export class RiffRaffYamlFile { deployments.set(lambdaDeployment.name, lambdaDeployment.props); }); - autoscalingGroups.forEach((asg) => { + // ASGs without an UpdatePolicy can be deployed via Riff-Raff's (legacy) `autoscaling` deployment type. + // ASGs with an UpdatePolicy are updated via Riff-Raff's `cloud-formation` deployment type. + const legacyAutoscalingGroups = autoscalingGroups.filter((asg) => { + const { cfnOptions } = asg.node.defaultChild as CfnAutoScalingGroup; + const { updatePolicy } = cfnOptions; + return ( + updatePolicy?.autoScalingReplacingUpdate === undefined && + updatePolicy?.autoScalingRollingUpdate === undefined + ); + }); + + legacyAutoscalingGroups.forEach((asg) => { const asgDeployment = autoscalingDeployment(asg, cfnDeployment); deployments.set(asgDeployment.name, asgDeployment.props); }); From 24df0b2860cee15d4926c9a74f76a1293d2db562 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Wed, 24 Jul 2024 07:48:35 +0100 Subject: [PATCH 02/23] feat(experimental-ec2-pattern): Add pattern to deploy ASGs updates via CloudFormation (`AutoScalingRollingUpdate`) --- .../__snapshots__/ec2-app.test.ts.snap | 1000 +++++++++++++++++ src/experimental/patterns/ec2-app.test.ts | 105 ++ src/experimental/patterns/ec2-app.ts | 111 ++ 3 files changed, 1216 insertions(+) create mode 100644 src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap create mode 100644 src/experimental/patterns/ec2-app.test.ts create mode 100644 src/experimental/patterns/ec2-app.ts diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap new file mode 100644 index 0000000000..4eb45f5be6 --- /dev/null +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -0,0 +1,1000 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` +{ + "Metadata": { + "gu:cdk:constructs": [ + "GuStack", + "GuDistributionBucketParameter", + "GuVpcParameter", + "GuSubnetListParameter", + "GuSubnetListParameter", + "GuEc2AppExperimental", + "GuCertificate", + "GuInstanceRole", + "GuSsmSshPolicy", + "GuDescribeEC2Policy", + "GuLoggingStreamNameParameter", + "GuLogShippingPolicy", + "GuGetDistributablePolicy", + "GuParameterStoreReadPolicy", + "GuAmiParameter", + "GuHttpsEgressSecurityGroup", + "GuWazuhAccess", + "GuAutoScalingGroup", + "GuApplicationLoadBalancer", + "GuApplicationTargetGroup", + "GuHttpsApplicationListener", + ], + "gu:cdk:version": "TEST", + }, + "Outputs": { + "LoadBalancerTestguec2appDnsName": { + "Description": "DNS entry for LoadBalancerTestguec2app", + "Value": { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appC77A055C", + "DNSName", + ], + }, + }, + }, + "Parameters": { + "AMITestguec2app": { + "Description": "Amazon Machine Image ID for the app test-gu-ec2-app. Use this in conjunction with AMIgo to keep AMIs up to date.", + "Type": "AWS::EC2::Image::Id", + }, + "DistributionBucketName": { + "Default": "/account/services/artifact.bucket", + "Description": "SSM parameter containing the S3 bucket name holding distribution artifacts", + "Type": "AWS::SSM::Parameter::Value", + }, + "LoggingStreamName": { + "Default": "/account/services/logging.stream.name", + "Description": "SSM parameter containing the Name (not ARN) on the kinesis stream", + "Type": "AWS::SSM::Parameter::Value", + }, + "VpcId": { + "Default": "/account/vpc/primary/id", + "Description": "Virtual Private Cloud to run EC2 instances within. Should NOT be the account default VPC.", + "Type": "AWS::SSM::Parameter::Value", + }, + "testguec2appPrivateSubnets": { + "Default": "/account/vpc/primary/subnets/private", + "Description": "A list of private subnets", + "Type": "AWS::SSM::Parameter::Value>", + }, + "testguec2appPublicSubnets": { + "Default": "/account/vpc/primary/subnets/public", + "Description": "A list of public subnets", + "Type": "AWS::SSM::Parameter::Value>", + }, + }, + "Resources": { + "AsgReplacingUpdatePolicy78CF34D5": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "cloudformation:SignalResource", + "Effect": "Allow", + "Resource": { + "Ref": "AWS::StackId", + }, + }, + { + "Action": "elasticloadbalancing:DescribeTargetHealth", + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "AsgReplacingUpdatePolicy78CF34D5", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "AutoScalingGroupTestguec2appASG49EA1878": { + "CreationPolicy": { + "AutoScalingCreationPolicy": { + "MinSuccessfulInstancesPercent": 100, + }, + "ResourceSignal": { + "Count": 1, + "Timeout": "PT5M", + }, + }, + "Properties": { + "HealthCheckGracePeriod": 120, + "HealthCheckType": "ELB", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "teststackTESTtestguec2appAA7F41BE", + }, + "Version": { + "Fn::GetAtt": [ + "teststackTESTtestguec2appAA7F41BE", + "LatestVersionNumber", + ], + }, + }, + "MaxSize": "2", + "MetricsCollection": [ + { + "Granularity": "1Minute", + }, + ], + "MinSize": "1", + "Tags": [ + { + "Key": "App", + "PropagateAtLaunch": true, + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "PropagateAtLaunch": true, + "Value": "TEST", + }, + { + "Key": "gu:repo", + "PropagateAtLaunch": true, + "Value": "guardian/cdk", + }, + { + "Key": "LogKinesisStreamName", + "PropagateAtLaunch": true, + "Value": { + "Ref": "LoggingStreamName", + }, + }, + { + "Key": "Stack", + "PropagateAtLaunch": true, + "Value": "test-stack", + }, + { + "Key": "Stage", + "PropagateAtLaunch": true, + "Value": "TEST", + }, + ], + "TargetGroupARNs": [ + { + "Ref": "TargetGroupTestguec2app9F67D503", + }, + ], + "VPCZoneIdentifier": { + "Ref": "testguec2appPrivateSubnets", + }, + }, + "Type": "AWS::AutoScaling::AutoScalingGroup", + "UpdatePolicy": { + "AutoScalingRollingUpdate": { + "MaxBatchSize": 2, + "MinInstancesInService": 1, + "MinSuccessfulInstancesPercent": 100, + "SuspendProcesses": [], + "WaitOnResourceSignals": true, + }, + "AutoScalingScheduledAction": { + "IgnoreUnmodifiedGroupSizeProperties": true, + }, + }, + }, + "CertificateTestguec2app86EE2D42": { + "DeletionPolicy": "Retain", + "Properties": { + "DomainName": "domain-name-for-your-application.example", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Name", + "Value": "Test/CertificateTestguec2app", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "ValidationMethod": "DNS", + }, + "Type": "AWS::CertificateManager::Certificate", + "UpdateReplacePolicy": "Retain", + }, + "DescribeEC2PolicyFF5F9295": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeAutoScalingGroups", + "ec2:DescribeTags", + "ec2:DescribeInstances", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "describe-ec2-policy", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "GetDistributablePolicyTestguec2app6A8D1854": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:s3:::", + { + "Ref": "DistributionBucketName", + }, + "/test-stack/TEST/test-gu-ec2-app/*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "GetDistributablePolicyTestguec2app6A8D1854", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "GuHttpsEgressSecurityGroupTestguec2appEBD7B195": { + "Properties": { + "GroupDescription": "Allow all outbound HTTPS traffic", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound HTTPS traffic", + "FromPort": 443, + "IpProtocol": "tcp", + "ToPort": 443, + }, + ], + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "VpcId": { + "Ref": "VpcId", + }, + }, + "Type": "AWS::EC2::SecurityGroup", + }, + "GuHttpsEgressSecurityGroupTestguec2appfromTestLoadBalancerTestguec2appSecurityGroup5F9E11C99000A9F74C7B": { + "Properties": { + "Description": "Load balancer to target", + "FromPort": 9000, + "GroupId": { + "Fn::GetAtt": [ + "GuHttpsEgressSecurityGroupTestguec2appEBD7B195", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appSecurityGroupCC6F85C1", + "GroupId", + ], + }, + "ToPort": 9000, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "GuLogShippingPolicy981BFE5A": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:Describe*", + "kinesis:Put*", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:kinesis:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stream/", + { + "Ref": "LoggingStreamName", + }, + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "GuLogShippingPolicy981BFE5A", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "InstanceRoleTestguec2appC325BE42": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Path": "/", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ListenerTestguec2app4FBB034F": { + "Properties": { + "Certificates": [ + { + "CertificateArn": { + "Ref": "CertificateTestguec2app86EE2D42", + }, + }, + ], + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "TargetGroupTestguec2app9F67D503", + }, + "Type": "forward", + }, + ], + "LoadBalancerArn": { + "Ref": "LoadBalancerTestguec2appC77A055C", + }, + "Port": 443, + "Protocol": "HTTPS", + "SslPolicy": "ELBSecurityPolicy-TLS13-1-2-2021-06", + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener", + }, + "LoadBalancerTestguec2appC77A055C": { + "Properties": { + "LoadBalancerAttributes": [ + { + "Key": "deletion_protection.enabled", + "Value": "true", + }, + { + "Key": "routing.http.x_amzn_tls_version_and_cipher_suite.enabled", + "Value": "true", + }, + { + "Key": "routing.http.drop_invalid_header_fields.enabled", + "Value": "true", + }, + ], + "Scheme": "internet-facing", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appSecurityGroupCC6F85C1", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "testguec2appPublicSubnets", + }, + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "Type": "application", + }, + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + }, + "LoadBalancerTestguec2appSecurityGroupCC6F85C1": { + "Properties": { + "GroupDescription": "Automatically created Security Group for ELB TestLoadBalancerTestguec2app8CD12AE9", + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow from anyone on port 443", + "FromPort": 443, + "IpProtocol": "tcp", + "ToPort": 443, + }, + ], + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "VpcId": { + "Ref": "VpcId", + }, + }, + "Type": "AWS::EC2::SecurityGroup", + }, + "LoadBalancerTestguec2appSecurityGrouptoTestGuHttpsEgressSecurityGroupTestguec2appE5EE51F5900063A5B571": { + "Properties": { + "Description": "Load balancer to target", + "DestinationSecurityGroupId": { + "Fn::GetAtt": [ + "GuHttpsEgressSecurityGroupTestguec2appEBD7B195", + "GroupId", + ], + }, + "FromPort": 9000, + "GroupId": { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appSecurityGroupCC6F85C1", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "ToPort": 9000, + }, + "Type": "AWS::EC2::SecurityGroupEgress", + }, + "LoadBalancerTestguec2appSecurityGrouptoTestWazuhSecurityGroup8092AEDC9000720EFF26": { + "Properties": { + "Description": "Load balancer to target", + "DestinationSecurityGroupId": { + "Fn::GetAtt": [ + "WazuhSecurityGroup", + "GroupId", + ], + }, + "FromPort": 9000, + "GroupId": { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appSecurityGroupCC6F85C1", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "ToPort": 9000, + }, + "Type": "AWS::EC2::SecurityGroupEgress", + }, + "ParameterStoreReadTestguec2app072DCDE1": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ssm:GetParametersByPath", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:ssm:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":parameter/TEST/test-stack/test-gu-ec2-app", + ], + ], + }, + }, + { + "Action": [ + "ssm:GetParameters", + "ssm:GetParameter", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:ssm:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":parameter/TEST/test-stack/test-gu-ec2-app/*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "parameter-store-read-policy", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "SsmSshPolicy4CFC977E": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "ssm:UpdateInstanceInformation", + "ssm:ListInstanceAssociations", + "ssm:DescribeInstanceProperties", + "ssm:DescribeDocumentParameters", + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ssm-ssh-policy", + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "TargetGroupTestguec2app9F67D503": { + "Properties": { + "HealthCheckIntervalSeconds": 10, + "HealthCheckPath": "/healthcheck", + "HealthCheckProtocol": "HTTP", + "HealthCheckTimeoutSeconds": 5, + "HealthyThresholdCount": 5, + "Port": 9000, + "Protocol": "HTTP", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "TargetGroupAttributes": [ + { + "Key": "deregistration_delay.timeout_seconds", + "Value": "30", + }, + { + "Key": "stickiness.enabled", + "Value": "false", + }, + ], + "TargetType": "instance", + "UnhealthyThresholdCount": 2, + "VpcId": { + "Ref": "VpcId", + }, + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + }, + "WazuhSecurityGroup": { + "Properties": { + "GroupDescription": "Allow outbound traffic from wazuh agent to manager", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Wazuh event logging", + "FromPort": 1514, + "IpProtocol": "tcp", + "ToPort": 1514, + }, + { + "CidrIp": "0.0.0.0/0", + "Description": "Wazuh agent registration", + "FromPort": 1515, + "IpProtocol": "tcp", + "ToPort": 1515, + }, + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "VpcId": { + "Ref": "VpcId", + }, + }, + "Type": "AWS::EC2::SecurityGroup", + }, + "WazuhSecurityGroupfromTestLoadBalancerTestguec2appSecurityGroup5F9E11C99000BB163DB4": { + "Properties": { + "Description": "Load balancer to target", + "FromPort": 9000, + "GroupId": { + "Fn::GetAtt": [ + "WazuhSecurityGroup", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "LoadBalancerTestguec2appSecurityGroupCC6F85C1", + "GroupId", + ], + }, + "ToPort": 9000, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "teststackTESTtestguec2appAA7F41BE": { + "DependsOn": [ + "InstanceRoleTestguec2appC325BE42", + ], + "Properties": { + "LaunchTemplateData": { + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "teststackTESTtestguec2appProfileC5759753", + "Arn", + ], + }, + }, + "ImageId": { + "Ref": "AMITestguec2app", + }, + "InstanceType": "t4g.medium", + "MetadataOptions": { + "HttpTokens": "required", + "InstanceMetadataTags": "enabled", + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "GuHttpsEgressSecurityGroupTestguec2appEBD7B195", + "GroupId", + ], + }, + { + "Fn::GetAtt": [ + "WazuhSecurityGroup", + "GroupId", + ], + }, + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Name", + "Value": "Test/test-stack-TEST-test-gu-ec2-app", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Name", + "Value": "Test/test-stack-TEST-test-gu-ec2-app", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash +function exitTrap(){ +exitCode=$? + + cfn-signal --stack ", + { + "Ref": "AWS::StackId", + }, + " --resource AutoScalingGroupTestguec2appASG49EA1878 --region ", + { + "Ref": "AWS::Region", + }, + " --exit-code $exitCode || echo 'Failed to send Cloudformation Signal' + +} +trap exitTrap EXIT +mkdir -p $(dirname '/test-gu-ec2-app/test-gu-ec2-app-123.deb') +aws s3 cp 's3://", + { + "Ref": "DistributionBucketName", + }, + "/test-stack/TEST/test-gu-ec2-app/test-gu-ec2-app-123.deb' '/test-gu-ec2-app/test-gu-ec2-app-123.deb' +dpkg -i /test-gu-ec2-app/test-gu-ec2-app-123.deb + + INSTANCE_ID=$(ec2metadata --instance-id) + + STATE=$(aws elbv2 describe-target-health --target-group-arn ", + { + "Ref": "TargetGroupTestguec2app9F67D503", + }, + " --region ", + { + "Ref": "AWS::Region", + }, + " --targets Id=$INSTANCE_ID,Port=9000 --query "TargetHealthDescriptions[0].TargetHealth.State") + + until [ "$STATE" == "\\"healthy\\"" ]; do + echo "Instance not yet healthy within target group. Current state $STATE. Sleeping..." + sleep 5 + STATE=$(aws elbv2 describe-target-health --target-group-arn ", + { + "Ref": "TargetGroupTestguec2app9F67D503", + }, + " --region ", + { + "Ref": "AWS::Region", + }, + " --targets Id=$INSTANCE_ID,Port=9000 --query "TargetHealthDescriptions[0].TargetHealth.State") + done + + echo "Instance is healthy in target group." + ", + ], + ], + }, + }, + }, + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "App", + "Value": "test-gu-ec2-app", + }, + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/cdk", + }, + { + "Key": "Name", + "Value": "Test/test-stack-TEST-test-gu-ec2-app", + }, + { + "Key": "Stack", + "Value": "test-stack", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + ], + }, + "Type": "AWS::EC2::LaunchTemplate", + }, + "teststackTESTtestguec2appProfileC5759753": { + "Properties": { + "Roles": [ + { + "Ref": "InstanceRoleTestguec2appC325BE42", + }, + ], + }, + "Type": "AWS::IAM::InstanceProfile", + }, + }, +} +`; diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts new file mode 100644 index 0000000000..14fc4665a1 --- /dev/null +++ b/src/experimental/patterns/ec2-app.test.ts @@ -0,0 +1,105 @@ +import { Duration } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; +import { InstanceClass, InstanceSize, InstanceType } from "aws-cdk-lib/aws-ec2"; +import { AccessScope } from "../../constants"; +import { GuUserData } from "../../constructs/autoscaling"; +import type { GuStack } from "../../constructs/core"; +import { simpleGuStackForTesting } from "../../utils/test"; +import type { GuEc2AppExperimentalProps } from "./ec2-app"; +import { GuEc2AppExperimental } from "./ec2-app"; + +// TODO test User Data includes a build number +describe("The GuEc2AppExperimental pattern", () => { + function initialProps(scope: GuStack): GuEc2AppExperimentalProps { + const app = "test-gu-ec2-app"; + const buildNumber = 123; + + const { userData } = new GuUserData(scope, { + app, + distributable: { + fileName: `${app}-${buildNumber}.deb`, + executionStatement: `dpkg -i /${app}/${app}-${buildNumber}.deb`, + }, + }); + + return { + applicationPort: 9000, + app, + access: { scope: AccessScope.PUBLIC }, + instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.MEDIUM), + monitoringConfiguration: { noMonitoring: true }, + userData, + certificateProps: { + domainName: "domain-name-for-your-application.example", + }, + scaling: { + minimumInstances: 1, + }, + }; + } + + it("matches the snapshot", () => { + const stack = simpleGuStackForTesting(); + new GuEc2AppExperimental(stack, initialProps(stack)); + expect(Template.fromStack(stack).toJSON()).toMatchSnapshot(); + }); + + it("should create an ASG with a resource signal count that matches the min instances", () => { + const stack = simpleGuStackForTesting(); + + new GuEc2AppExperimental(stack, { ...initialProps(stack), scaling: { minimumInstances: 5 } }); + + Template.fromStack(stack).hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "5", + }, + CreationPolicy: { + AutoScalingCreationPolicy: { + MinSuccessfulInstancesPercent: 100, + }, + ResourceSignal: { + Count: 5, + Timeout: "PT5M", + }, + }, + }); + }); + + it("should create an ASG with the maximum resource signal timeout", () => { + const stack = simpleGuStackForTesting(); + + const targetGroupHealthcheckTimeout = Duration.minutes(7); + + new GuEc2AppExperimental(stack, { + ...initialProps(stack), + healthcheck: { + timeout: targetGroupHealthcheckTimeout, + interval: Duration.seconds(targetGroupHealthcheckTimeout.toSeconds() + 60), + }, + }); + + const template = Template.fromStack(stack); + + // The Target Group times out in 7 minutes. + template.hasResourceProperties("AWS::ElasticLoadBalancingV2::TargetGroup", { + HealthCheckTimeoutSeconds: targetGroupHealthcheckTimeout.toSeconds(), + }); + + // The ASG grace period is 2 minutes, which is less than the Target Group. + // Therefore, the resource signal timeout should be 7 minutes. + template.hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + HealthCheckGracePeriod: 120, + }, + CreationPolicy: { + AutoScalingCreationPolicy: { + MinSuccessfulInstancesPercent: 100, + }, + ResourceSignal: { + Count: 1, + Timeout: targetGroupHealthcheckTimeout.toIsoString(), + }, + }, + }); + }); +}); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts new file mode 100644 index 0000000000..31d7384aea --- /dev/null +++ b/src/experimental/patterns/ec2-app.ts @@ -0,0 +1,111 @@ +import { Duration } from "aws-cdk-lib"; +import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; +import { UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; +import { Effect, Policy, PolicyStatement } from "aws-cdk-lib/aws-iam"; +import type { GuStack } from "../../constructs/core"; +import type { GuEc2AppProps } from "../../patterns"; +import { GuEc2App } from "../../patterns"; + +export interface GuEc2AppExperimentalProps extends Omit {} + +export class GuEc2AppExperimental extends GuEc2App { + constructor(scope: GuStack, props: GuEc2AppExperimentalProps) { + const { minimumInstances, maximumInstances = minimumInstances * 2 } = props.scaling; + const { applicationPort } = props; + const { region, stackId } = scope; + + super(scope, { + ...props, + updatePolicy: UpdatePolicy.rollingUpdate({ + maxBatchSize: maximumInstances, + minInstancesInService: minimumInstances, + minSuccessPercentage: 100, + waitOnResourceSignals: true, + suspendProcesses: [], + }), + }); + + const { autoScalingGroup, targetGroup } = this; + const { userData, role } = autoScalingGroup; + const cfnAutoScalingGroup = autoScalingGroup.node.defaultChild as CfnAutoScalingGroup; + + new Policy(scope, "AsgReplacingUpdatePolicy", { + statements: [ + // Allow usage of command `cfn-signal`. + new PolicyStatement({ + actions: ["cloudformation:SignalResource"], + effect: Effect.ALLOW, + resources: [stackId], + }), + + /* + Allow usage of command `aws elbv2 describe-target-health`. + AWS Elastic Load Balancing does not support resource based policies, so the resource has to be `*` (any) here. + See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html + */ + new PolicyStatement({ + actions: ["elasticloadbalancing:DescribeTargetHealth"], + effect: Effect.ALLOW, + resources: ["*"], + }), + ], + }).attachToRole(role); + + /* + `ec2metadata` is available via `cloud-utils` installed on all Canonical Ubuntu AMIs. + See https://github.com/canonical/cloud-utils. + + `aws` is available via AMIgo baked AMIs. + See https://github.com/guardian/amigo/tree/main/roles/aws-tools. + */ + userData.addCommands( + ` + INSTANCE_ID=$(ec2metadata --instance-id) + + STATE=$(aws elbv2 describe-target-health \ + --target-group-arn ${targetGroup.targetGroupArn} \ + --region ${region} \ + --targets Id=$INSTANCE_ID,Port=${applicationPort} \ + --query "TargetHealthDescriptions[0].TargetHealth.State") + + until [ "$STATE" == "\\"healthy\\"" ]; do + echo "Instance not yet healthy within target group. Current state $STATE. Sleeping..." + sleep 5 + STATE=$(aws elbv2 describe-target-health \ + --target-group-arn ${targetGroup.targetGroupArn} \ + --region ${region} \ + --targets Id=$INSTANCE_ID,Port=${applicationPort} \ + --query "TargetHealthDescriptions[0].TargetHealth.State") + done + + echo "Instance is healthy in target group." + `, + ); + + userData.addOnExitCommands( + ` + cfn-signal --stack ${stackId} \ + --resource ${cfnAutoScalingGroup.logicalId} \ + --region ${region} \ + --exit-code $exitCode || echo 'Failed to send Cloudformation Signal' + `, + ); + + // TODO are these sensible values? + const signalTimeoutSeconds = Math.max( + targetGroup.healthCheck.timeout?.toSeconds() ?? 0, + cfnAutoScalingGroup.healthCheckGracePeriod ?? 0, + Duration.minutes(5).toSeconds(), + ); + + cfnAutoScalingGroup.cfnOptions.creationPolicy = { + autoScalingCreationPolicy: { + minSuccessfulInstancesPercent: 100, + }, + resourceSignal: { + count: minimumInstances, + timeout: Duration.seconds(signalTimeoutSeconds).toIsoString(), + }, + }; + } +} From 10701fd0ce2c56c0555f99ac663d16e333172558 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Wed, 24 Jul 2024 07:49:25 +0100 Subject: [PATCH 03/23] test(riff-raff.yaml): Test `GuEc2AppExperimental` behaviour --- src/riff-raff-yaml-file/index.test.ts | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/riff-raff-yaml-file/index.test.ts b/src/riff-raff-yaml-file/index.test.ts index 3a19a72f2f..3b9a33748d 100644 --- a/src/riff-raff-yaml-file/index.test.ts +++ b/src/riff-raff-yaml-file/index.test.ts @@ -7,6 +7,7 @@ import { AccessScope } from "../constants"; import type { GuStackProps } from "../constructs/core"; import { GuStack } from "../constructs/core"; import { GuLambdaFunction } from "../constructs/lambda"; +import { GuEc2AppExperimental } from "../experimental/patterns/ec2-app"; import { GuEc2App, GuNodeApp, GuPlayApp, GuScheduledLambda } from "../patterns"; import { RiffRaffYamlFile } from "./index"; @@ -1334,4 +1335,81 @@ describe("The RiffRaffYamlFile class", () => { " `); }); + + it("Should only upload artifacts for a GuEc2AppExperimental", () => { + const app = new App({ outdir: "/tmp/cdk.out" }); + + class MyApplicationStack extends GuStack { + // eslint-disable-next-line custom-rules/valid-constructors -- unit testing + constructor(app: App, id: string, props: GuStackProps) { + super(app, id, props); + + const appName = "my-app"; + + new GuEc2AppExperimental(this, { + app: appName, + instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.MICRO), + access: { scope: AccessScope.PUBLIC }, + userData: { + distributable: { + fileName: `${appName}.deb`, + executionStatement: `dpkg -i /${appName}/${appName}.deb`, + }, + }, + certificateProps: { + domainName: "rip.gu.com", + }, + monitoringConfiguration: { noMonitoring: true }, + scaling: { + minimumInstances: 1, + }, + applicationPort: 9000, + imageRecipe: "arm64-bionic-java11-deploy-infrastructure", + }); + } + } + + new MyApplicationStack(app, "test-stack", { stack: "test", stage: "TEST", env: { region: "eu-west-1" } }); + + const actual = new RiffRaffYamlFile(app).toYAML(); + + expect(actual).toMatchInlineSnapshot(` + "allowedStages: + - TEST + deployments: + asg-upload-eu-west-1-test-my-app: + type: autoscaling + actions: + - uploadArtifacts + regions: + - eu-west-1 + stacks: + - test + app: my-app + parameters: + bucketSsmLookup: true + prefixApp: true + contentDirectory: my-app + cfn-eu-west-1-test-my-application-stack: + type: cloud-formation + regions: + - eu-west-1 + stacks: + - test + app: my-application-stack + contentDirectory: /tmp/cdk.out + parameters: + templateStagePaths: + TEST: test-stack.template.json + amiParametersToTags: + AMIMyapp: + BuiltBy: amigo + AmigoStage: PROD + Recipe: arm64-bionic-java11-deploy-infrastructure + Encrypted: 'true' + dependencies: + - asg-upload-eu-west-1-test-my-app + " + `); + }); }); From c83449c3f3c15ba5c5c7f9cc5816e80e9ef1b9a8 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Wed, 24 Jul 2024 11:45:26 +0100 Subject: [PATCH 04/23] feat(experimental-ec2-pattern): Decorate added user data commands w/markers This should make it easier to parse a user data string if ever one is debugging. --- .../__snapshots__/ec2-app.test.ts.snap | 4 ++- src/experimental/patterns/ec2-app.test.ts | 26 ++++++++++++++++++- src/experimental/patterns/ec2-app.ts | 2 ++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index 4eb45f5be6..f177b640cc 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -917,6 +917,7 @@ aws s3 cp 's3://", }, "/test-stack/TEST/test-gu-ec2-app/test-gu-ec2-app-123.deb' '/test-gu-ec2-app/test-gu-ec2-app-123.deb' dpkg -i /test-gu-ec2-app/test-gu-ec2-app-123.deb +# GuEc2AppExperimental UserData Start INSTANCE_ID=$(ec2metadata --instance-id) @@ -945,7 +946,8 @@ dpkg -i /test-gu-ec2-app/test-gu-ec2-app-123.deb done echo "Instance is healthy in target group." - ", + +# GuEc2AppExperimental UserData End", ], ], }, diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 14fc4665a1..9194979e2e 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -1,6 +1,6 @@ import { Duration } from "aws-cdk-lib"; import { Template } from "aws-cdk-lib/assertions"; -import { InstanceClass, InstanceSize, InstanceType } from "aws-cdk-lib/aws-ec2"; +import { InstanceClass, InstanceSize, InstanceType, UserData } from "aws-cdk-lib/aws-ec2"; import { AccessScope } from "../../constants"; import { GuUserData } from "../../constructs/autoscaling"; import type { GuStack } from "../../constructs/core"; @@ -102,4 +102,28 @@ describe("The GuEc2AppExperimental pattern", () => { }, }); }); + + it("should add to the end of the user data", () => { + const stack = simpleGuStackForTesting(); + + const userDataCommand = `echo "Hello there"`; + const userData = UserData.forLinux(); + userData.addCommands(userDataCommand); + + const { autoScalingGroup } = new GuEc2AppExperimental(stack, { ...initialProps(stack), userData }); + + const renderedUserData = autoScalingGroup.userData.render(); + const splitUserData = renderedUserData.split("\n"); + const totalLines = splitUserData.length; + + const appCommandPosition = splitUserData.indexOf(userDataCommand); + const startMarkerPosition = splitUserData.indexOf("# GuEc2AppExperimental UserData Start"); + const endMarkerPosition = splitUserData.indexOf("# GuEc2AppExperimental UserData End"); + + // Application user data should be before the target group healthcheck polling. + expect(appCommandPosition).toBeLessThan(startMarkerPosition); + + // The target group healthcheck polling should be the last thing in the user data. + expect(endMarkerPosition).toEqual(totalLines - 1); + }); }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 31d7384aea..430e6a8a20 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -59,6 +59,7 @@ export class GuEc2AppExperimental extends GuEc2App { See https://github.com/guardian/amigo/tree/main/roles/aws-tools. */ userData.addCommands( + `# ${GuEc2AppExperimental.name} UserData Start`, ` INSTANCE_ID=$(ec2metadata --instance-id) @@ -80,6 +81,7 @@ export class GuEc2AppExperimental extends GuEc2App { echo "Instance is healthy in target group." `, + `# ${GuEc2AppExperimental.name} UserData End`, ); userData.addOnExitCommands( From de842f6ed3ed7838932aaa3d3aab8860d1ee9706 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Mon, 29 Jul 2024 11:01:01 +0100 Subject: [PATCH 05/23] docs(experimental-ec2-pattern): Add doc string to class --- src/experimental/patterns/ec2-app.ts | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 430e6a8a20..9ebb606987 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -8,6 +8,35 @@ import { GuEc2App } from "../../patterns"; export interface GuEc2AppExperimentalProps extends Omit {} +/** + * An experimental pattern to instantiate an EC2 application that is updated entirely via CloudFormation. + * + * NOTE: The "autoscaling" deployment type in Riff-Raff is not valid with this pattern. + * Please remove it from any manually created `riff-raff.yaml` file. + * + * This pattern sets the update policy of the `AWS::AutoScaling::AutoScalingGroup` to `AutoScalingRollingUpdate`. + * When a CloudFormation update is applied, the current instances in the ASG will be replaced. + * + * This pattern also updates the User Data, running some commands AFTER yours. + * These changes are wrapped in start and end marking comments. + * + * This pattern should improve the reliability of scaling events triggered during a deployment. + * Unlike in Riff-Raff's "autoscaling" deployment, scaling alarms are never suspended. + * TODO test scaling alarm behaviour. + * + * To update the application's code, a CloudFormation update must be triggered. + * The best way to do this is to include the build number in the application artifact. + * TODO test User Data includes a build number. + * + * NOTE: This pattern: + * - Is NOT compatible with the "autoscaling" Riff-Raff deployment type. + * - Your application should include a build number in its filename. + * This value will change across builds, and therefore create a CloudFormation template difference to be deployed. + * - Requires the AWS CLI and `cfn-signal` binaries to be available on the instance, and on the `PATH`. + * AMIgo adds these via the `aws-tools` role. + * + * @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-attribute-updatepolicy.html#cfn-attributes-updatepolicy-rollingupdate + */ export class GuEc2AppExperimental extends GuEc2App { constructor(scope: GuStack, props: GuEc2AppExperimentalProps) { const { minimumInstances, maximumInstances = minimumInstances * 2 } = props.scaling; From 3fe10c623a78174c5a5071058f5fd5fa55883996 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 20 Aug 2024 21:41:37 +0100 Subject: [PATCH 06/23] fix(experimental-ec2-pattern): Set ASG `DesiredCapacity` During a deployment, CloudFormation updates the min and desired. During a rollback (e.g. if the healthcheck failed), CloudFormation only resets the min. The desired is still elevated, meaning the service is over provisioned. Explicitly setting the desired property of the ASG ensures CloudFormation rollback puts the ASG back to the initial state, e.g. correctly provisioned. --- .../patterns/__snapshots__/ec2-app.test.ts.snap | 1 + src/experimental/patterns/ec2-app.test.ts | 14 ++++++++++++++ src/experimental/patterns/ec2-app.ts | 2 ++ 3 files changed, 17 insertions(+) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index f177b640cc..5e6c980e15 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -110,6 +110,7 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` }, }, "Properties": { + "DesiredCapacity": "1", "HealthCheckGracePeriod": 120, "HealthCheckType": "ELB", "LaunchTemplate": { diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 9194979e2e..9d51a5808c 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -44,6 +44,20 @@ describe("The GuEc2AppExperimental pattern", () => { expect(Template.fromStack(stack).toJSON()).toMatchSnapshot(); }); + it("should create an ASG with min, max, and desired capacity set", () => { + const stack = simpleGuStackForTesting(); + + new GuEc2AppExperimental(stack, { ...initialProps(stack), scaling: { minimumInstances: 5 } }); + + Template.fromStack(stack).hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "5", + MaxSize: "10", + DesiredCapacity: "5", + }, + }); + }); + it("should create an ASG with a resource signal count that matches the min instances", () => { const stack = simpleGuStackForTesting(); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 9ebb606987..37ad015616 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -58,6 +58,8 @@ export class GuEc2AppExperimental extends GuEc2App { const { userData, role } = autoScalingGroup; const cfnAutoScalingGroup = autoScalingGroup.node.defaultChild as CfnAutoScalingGroup; + cfnAutoScalingGroup.desiredCapacity = minimumInstances.toString(); + new Policy(scope, "AsgReplacingUpdatePolicy", { statements: [ // Allow usage of command `cfn-signal`. From 6efd2dc1a7de7199722dbe180c70486377031ff7 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 10:16:55 +0100 Subject: [PATCH 07/23] refactor(experimental-ec2-pattern): Use the same duration when creating and updating ASG --- .../__snapshots__/ec2-app.test.ts.snap | 4 +- src/experimental/patterns/ec2-app.ts | 43 +++++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index 5e6c980e15..98db07b14b 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -180,12 +180,10 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` "MaxBatchSize": 2, "MinInstancesInService": 1, "MinSuccessfulInstancesPercent": 100, + "PauseTime": "PT5M", "SuspendProcesses": [], "WaitOnResourceSignals": true, }, - "AutoScalingScheduledAction": { - "IgnoreUnmodifiedGroupSizeProperties": true, - }, }, }, "CertificateTestguec2app86EE2D42": { diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 37ad015616..67ffe063af 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -60,6 +60,32 @@ export class GuEc2AppExperimental extends GuEc2App { cfnAutoScalingGroup.desiredCapacity = minimumInstances.toString(); + // TODO are these sensible values? + const signalTimeoutSeconds = Math.max( + targetGroup.healthCheck.timeout?.toSeconds() ?? 0, + cfnAutoScalingGroup.healthCheckGracePeriod ?? 0, + Duration.minutes(5).toSeconds(), + ); + + const currentRollingUpdate = cfnAutoScalingGroup.cfnOptions.updatePolicy?.autoScalingRollingUpdate; + + cfnAutoScalingGroup.cfnOptions.updatePolicy = { + autoScalingRollingUpdate: { + ...currentRollingUpdate, + pauseTime: Duration.seconds(signalTimeoutSeconds).toIsoString(), + }, + }; + + cfnAutoScalingGroup.cfnOptions.creationPolicy = { + autoScalingCreationPolicy: { + minSuccessfulInstancesPercent: 100, + }, + resourceSignal: { + count: minimumInstances, + timeout: Duration.seconds(signalTimeoutSeconds).toIsoString(), + }, + }; + new Policy(scope, "AsgReplacingUpdatePolicy", { statements: [ // Allow usage of command `cfn-signal`. @@ -123,22 +149,5 @@ export class GuEc2AppExperimental extends GuEc2App { --exit-code $exitCode || echo 'Failed to send Cloudformation Signal' `, ); - - // TODO are these sensible values? - const signalTimeoutSeconds = Math.max( - targetGroup.healthCheck.timeout?.toSeconds() ?? 0, - cfnAutoScalingGroup.healthCheckGracePeriod ?? 0, - Duration.minutes(5).toSeconds(), - ); - - cfnAutoScalingGroup.cfnOptions.creationPolicy = { - autoScalingCreationPolicy: { - minSuccessfulInstancesPercent: 100, - }, - resourceSignal: { - count: minimumInstances, - timeout: Duration.seconds(signalTimeoutSeconds).toIsoString(), - }, - }; } } From e089a8473d67c3f07f3f13610a5f0835c3f34ac7 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 10:35:14 +0100 Subject: [PATCH 08/23] fix(experimental-ec2-pattern): Suspend alarm notifications A scale-in event fires during a rolling update can cause service disruption. Suspend scaling events during a rolling update for safety. --- .../patterns/__snapshots__/ec2-app.test.ts.snap | 4 +++- src/experimental/patterns/ec2-app.ts | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index 98db07b14b..fc36444190 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -181,7 +181,9 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` "MinInstancesInService": 1, "MinSuccessfulInstancesPercent": 100, "PauseTime": "PT5M", - "SuspendProcesses": [], + "SuspendProcesses": [ + "AlarmNotification", + ], "WaitOnResourceSignals": true, }, }, diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 67ffe063af..3bef4d6dc2 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -50,7 +50,14 @@ export class GuEc2AppExperimental extends GuEc2App { minInstancesInService: minimumInstances, minSuccessPercentage: 100, waitOnResourceSignals: true, - suspendProcesses: [], + + /* + If a scale-in event fires during an `AutoScalingRollingUpdate` operation, the update could fail and rollback. + For this reason, we suspend the `AlarmNotification` process, else availability of a service cannot be guaranteed. + Consequently, services cannot scale-out during deployments. + If AWS ever supports suspending scale-out and scale-in independently, we should allow scale-out. + */ + suspendProcesses: [ScalingProcess.ALARM_NOTIFICATION], }), }); From b2d7782d6ff20d32e09ff0077cf92e3084ec9fa4 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 13:10:12 +0100 Subject: [PATCH 09/23] feat(experimental-ec2-pattern): Adjust ASG rolling update properties where scaling policy present Some practical testing of `AutoScalingRollingUpdate` has demonstrated that when an ASG has a scaling policy, it is safest to dynamically set the `MinInstancesInService` property. Add an aspect to do that. See also https://github.com/guardian/testing-asg-rolling-update. --- src/experimental/patterns/ec2-app.test.ts | 73 +++++++++++++++++++++-- src/experimental/patterns/ec2-app.ts | 62 ++++++++++++++++++- 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 9d51a5808c..f67d071fd4 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -1,17 +1,17 @@ -import { Duration } from "aws-cdk-lib"; -import { Template } from "aws-cdk-lib/assertions"; +import { App, Duration } from "aws-cdk-lib"; +import { Match, Template } from "aws-cdk-lib/assertions"; import { InstanceClass, InstanceSize, InstanceType, UserData } from "aws-cdk-lib/aws-ec2"; +import { CloudFormationStackArtifact } from "aws-cdk-lib/cx-api"; import { AccessScope } from "../../constants"; import { GuUserData } from "../../constructs/autoscaling"; -import type { GuStack } from "../../constructs/core"; +import { GuStack } from "../../constructs/core"; import { simpleGuStackForTesting } from "../../utils/test"; import type { GuEc2AppExperimentalProps } from "./ec2-app"; import { GuEc2AppExperimental } from "./ec2-app"; // TODO test User Data includes a build number describe("The GuEc2AppExperimental pattern", () => { - function initialProps(scope: GuStack): GuEc2AppExperimentalProps { - const app = "test-gu-ec2-app"; + function initialProps(scope: GuStack, app: string = "test-gu-ec2-app"): GuEc2AppExperimentalProps { const buildNumber = 123; const { userData } = new GuUserData(scope, { @@ -140,4 +140,67 @@ describe("The GuEc2AppExperimental pattern", () => { // The target group healthcheck polling should be the last thing in the user data. expect(endMarkerPosition).toEqual(totalLines - 1); }); + + it("should adjust properties of a horizontally scaling service", () => { + const cdkApp = new App(); + const stack = new GuStack(cdkApp, "test", { + stack: "test-stack", + stage: "TEST", + }); + + const scalingApp = "my-scaling-app"; + const { autoScalingGroup } = new GuEc2AppExperimental(stack, { + ...initialProps(stack, scalingApp), + scaling: { + minimumInstances: 5, + }, + }); + autoScalingGroup.scaleOnRequestCount("ScaleOnRequests", { + targetRequestsPerMinute: 100, + }); + + /* + We're ultimately testing an `Aspect`, which appear to run only at synth time. + As a work-around, synth the `App`, then perform assertions on the resulting template. + + See also: https://github.com/aws/aws-cdk/issues/29047. + */ + const { artifacts } = cdkApp.synth(); + const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); + + if (!cfnStack) { + throw new Error("Unable to locate a CloudFormationStackArtifact"); + } + + const template = Template.fromJSON(cfnStack.template as Record); + + const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; + + template.hasParameter(parameterName, { + Type: "Number", + Default: 5, + MaxValue: 9, // (min * 2) - 1 + }); + + template.hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "5", + MaxSize: "10", + DesiredCapacity: Match.absent(), + Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), + }, + UpdatePolicy: { + AutoScalingRollingUpdate: { + MaxBatchSize: 10, + SuspendProcesses: ["AlarmNotification"], + MinSuccessfulInstancesPercent: 100, + WaitOnResourceSignals: true, + PauseTime: "PT5M", + MinInstancesInService: { + Ref: parameterName, + }, + }, + }, + }); + }); }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 3bef4d6dc2..9c294daf66 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -1,11 +1,65 @@ -import { Duration } from "aws-cdk-lib"; +import type { IAspect } from "aws-cdk-lib"; +import { Aspects, CfnParameter, Duration } from "aws-cdk-lib"; import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; -import { UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; +import { CfnScalingPolicy, ScalingProcess, UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; import { Effect, Policy, PolicyStatement } from "aws-cdk-lib/aws-iam"; -import type { GuStack } from "../../constructs/core"; +import type { IConstruct } from "constructs"; +import { GuAutoScalingGroup } from "../../constructs/autoscaling"; +import { GuStack } from "../../constructs/core"; import type { GuEc2AppProps } from "../../patterns"; import { GuEc2App } from "../../patterns"; +class HorizontallyScalingDeploymentProperties implements IAspect { + public visit(construct: IConstruct) { + if (construct instanceof CfnScalingPolicy) { + const { node } = construct; + const { scopes, path } = node; + const guStack = GuStack.of(construct); + + const autoScalingGroup = scopes.find((_): _ is GuAutoScalingGroup => _ instanceof GuAutoScalingGroup); + + if (!autoScalingGroup) { + throw new Error(`Failed to detect the autoscaling group relating to the scaling policy on path ${path}`); + } + + const cfnAutoScalingGroup = autoScalingGroup.node.defaultChild as CfnAutoScalingGroup; + const currentRollingUpdate = cfnAutoScalingGroup.cfnOptions.updatePolicy?.autoScalingRollingUpdate; + + if (currentRollingUpdate) { + /* + An autoscaling group that horizontally scales should not explicitly set `Desired`, + as a rolling update will set the current `Desired` back to the template version, + undoing any changes that a scale-out event may have done. + */ + cfnAutoScalingGroup.desiredCapacity = undefined; + + /* + An autoscaling group that horizontally scales should expose a CloudFormation Parameter linked to the + `MinInstancesInService` property of the rolling update policy. + + Riff-Raff will set this parameter during deployment. + The value depends on the current capacity of the ASG: + - If the service is running normally, it'll be set to the `Minimum` capacity + - If the service is partially scaled, it'll be set to the current `Desired` capacity + - If the service is fully scaled, it'll be set to (at least) `Maximum` - 1 + */ + const minInstancesInService = new CfnParameter(guStack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { + type: "Number", + default: parseInt(cfnAutoScalingGroup.minSize), + maxValue: parseInt(cfnAutoScalingGroup.maxSize) - 1, + }); + + cfnAutoScalingGroup.cfnOptions.updatePolicy = { + autoScalingRollingUpdate: { + ...currentRollingUpdate, + minInstancesInService: minInstancesInService.valueAsNumber, + }, + }; + } + } + } +} + export interface GuEc2AppExperimentalProps extends Omit {} /** @@ -156,5 +210,7 @@ export class GuEc2AppExperimental extends GuEc2App { --exit-code $exitCode || echo 'Failed to send Cloudformation Signal' `, ); + + Aspects.of(scope).add(new HorizontallyScalingDeploymentProperties()); } } From 22c9e48d40588bf643b45c92dc5e410b6b7f3525 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 14:17:14 +0100 Subject: [PATCH 10/23] test(experimental-ec2-pattern): Ensure only a horizontally scaling ASG is adjusted --- .../__snapshots__/ec2-app.test.ts.snap | 4 +- src/experimental/patterns/ec2-app.test.ts | 93 +++++++++++++++++++ src/experimental/patterns/ec2-app.ts | 80 +++++++++++----- 3 files changed, 152 insertions(+), 25 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index fc36444190..1f383c074f 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -71,7 +71,7 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` }, }, "Resources": { - "AsgReplacingUpdatePolicy78CF34D5": { + "AsgRollingUpdatePolicy2A1DDC6F": { "Properties": { "PolicyDocument": { "Statement": [ @@ -90,7 +90,7 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` ], "Version": "2012-10-17", }, - "PolicyName": "AsgReplacingUpdatePolicy78CF34D5", + "PolicyName": "AsgRollingUpdatePolicy2A1DDC6F", "Roles": [ { "Ref": "InstanceRoleTestguec2appC325BE42", diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index f67d071fd4..5eae6792bf 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -203,4 +203,97 @@ describe("The GuEc2AppExperimental pattern", () => { }, }); }); + + it("should only adjust properties of a horizontally scaling service", () => { + const cdkApp = new App(); + const stack = new GuStack(cdkApp, "test", { + stack: "test-stack", + stage: "TEST", + }); + + const scalingApp = "my-scaling-app"; + const { autoScalingGroup } = new GuEc2AppExperimental(stack, { + ...initialProps(stack, scalingApp), + scaling: { + minimumInstances: 5, + }, + }); + autoScalingGroup.scaleOnRequestCount("ScaleOnRequests", { + targetRequestsPerMinute: 100, + }); + + const staticApp = "my-static-app"; + new GuEc2AppExperimental(stack, initialProps(stack, staticApp)); + + /* + We're ultimately testing an `Aspect`, which appear to run only at synth time. + As a work-around, synth the `App`, then perform assertions on the resulting template. + + See also: https://github.com/aws/aws-cdk/issues/29047. + */ + const { artifacts } = cdkApp.synth(); + const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); + + if (!cfnStack) { + throw new Error("Unable to locate a CloudFormationStackArtifact"); + } + + const template = Template.fromJSON(cfnStack.template as Record); + + /* + The scaling ASG should: + - Not have `DesiredCapacity` set + - Have `MinInstancesInService` set via a CFN Parameter + */ + const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; + template.hasParameter(parameterName, { + Type: "Number", + Default: 5, + MaxValue: 9, // (min * 2) - 1 + }); + template.hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "5", + MaxSize: "10", + DesiredCapacity: Match.absent(), + Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), + }, + UpdatePolicy: { + AutoScalingRollingUpdate: { + MaxBatchSize: 10, + SuspendProcesses: ["AlarmNotification"], + MinSuccessfulInstancesPercent: 100, + WaitOnResourceSignals: true, + PauseTime: "PT5M", + MinInstancesInService: { + Ref: parameterName, + }, + }, + }, + }); + + /* + The static ASG should: + - Have `DesiredCapacity` set explicitly + - Have `MinInstancesInService` set explicitly + */ + template.hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "1", + MaxSize: "2", + DesiredCapacity: "1", + Tags: Match.arrayWith([{ Key: "App", Value: staticApp, PropagateAtLaunch: true }]), + }, + UpdatePolicy: { + AutoScalingRollingUpdate: { + MaxBatchSize: 2, + SuspendProcesses: ["AlarmNotification"], + MinSuccessfulInstancesPercent: 100, + WaitOnResourceSignals: true, + PauseTime: "PT5M", + MinInstancesInService: 1, + }, + }, + }); + }); }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 9c294daf66..fb4876ac03 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -1,4 +1,4 @@ -import type { IAspect } from "aws-cdk-lib"; +import type { IAspect, Stack } from "aws-cdk-lib"; import { Aspects, CfnParameter, Duration } from "aws-cdk-lib"; import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; import { CfnScalingPolicy, ScalingProcess, UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; @@ -8,8 +8,24 @@ import { GuAutoScalingGroup } from "../../constructs/autoscaling"; import { GuStack } from "../../constructs/core"; import type { GuEc2AppProps } from "../../patterns"; import { GuEc2App } from "../../patterns"; +import { isSingletonPresentInStack } from "../../utils/singleton"; class HorizontallyScalingDeploymentProperties implements IAspect { + public readonly stack: Stack; + private static instance: HorizontallyScalingDeploymentProperties | undefined; + + private constructor(scope: GuStack) { + this.stack = scope; + } + + public static getInstance(stack: GuStack): HorizontallyScalingDeploymentProperties { + if (!this.instance || !isSingletonPresentInStack(stack, this.instance)) { + this.instance = new HorizontallyScalingDeploymentProperties(stack); + } + + return this.instance; + } + public visit(construct: IConstruct) { if (construct instanceof CfnScalingPolicy) { const { node } = construct; @@ -60,6 +76,44 @@ class HorizontallyScalingDeploymentProperties implements IAspect { } } +class AsgRollingUpdatePolicy extends Policy { + private static instance: AsgRollingUpdatePolicy | undefined; + + private constructor(scope: GuStack) { + const { stackId } = scope; + + super(scope, "AsgRollingUpdatePolicy", { + statements: [ + // Allow usage of command `cfn-signal`. + new PolicyStatement({ + actions: ["cloudformation:SignalResource"], + effect: Effect.ALLOW, + resources: [stackId], + }), + + /* + Allow usage of command `aws elbv2 describe-target-health`. + AWS Elastic Load Balancing does not support resource based policies, so the resource has to be `*` (any) here. + See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html + */ + new PolicyStatement({ + actions: ["elasticloadbalancing:DescribeTargetHealth"], + effect: Effect.ALLOW, + resources: ["*"], + }), + ], + }); + } + + public static getInstance(stack: GuStack): AsgRollingUpdatePolicy { + if (!this.instance || !isSingletonPresentInStack(stack, this.instance)) { + this.instance = new AsgRollingUpdatePolicy(stack); + } + + return this.instance; + } +} + export interface GuEc2AppExperimentalProps extends Omit {} /** @@ -147,27 +201,7 @@ export class GuEc2AppExperimental extends GuEc2App { }, }; - new Policy(scope, "AsgReplacingUpdatePolicy", { - statements: [ - // Allow usage of command `cfn-signal`. - new PolicyStatement({ - actions: ["cloudformation:SignalResource"], - effect: Effect.ALLOW, - resources: [stackId], - }), - - /* - Allow usage of command `aws elbv2 describe-target-health`. - AWS Elastic Load Balancing does not support resource based policies, so the resource has to be `*` (any) here. - See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html - */ - new PolicyStatement({ - actions: ["elasticloadbalancing:DescribeTargetHealth"], - effect: Effect.ALLOW, - resources: ["*"], - }), - ], - }).attachToRole(role); + AsgRollingUpdatePolicy.getInstance(scope).attachToRole(role); /* `ec2metadata` is available via `cloud-utils` installed on all Canonical Ubuntu AMIs. @@ -211,6 +245,6 @@ export class GuEc2AppExperimental extends GuEc2App { `, ); - Aspects.of(scope).add(new HorizontallyScalingDeploymentProperties()); + Aspects.of(scope).add(HorizontallyScalingDeploymentProperties.getInstance(scope)); } } From 5ce18cd2e823118e7134277acf8b257b3bcd022c Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 15:31:17 +0100 Subject: [PATCH 11/23] docs(experimental-ec2-pattern): Update doc strings --- src/experimental/patterns/ec2-app.ts | 70 ++++++++++++++++++---------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index fb4876ac03..203f4b7fdd 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -10,6 +10,26 @@ import type { GuEc2AppProps } from "../../patterns"; import { GuEc2App } from "../../patterns"; import { isSingletonPresentInStack } from "../../utils/singleton"; +/** + * An `Aspect` that adjusts the properties of an AutoScaling Group using an `AutoScalingRollingUpdate` update policy. + * + * It'll unset the `DesiredCapacity` of the ASG as a rolling update sets `Desired` back to the template version, + * undoing any changes that a scale-out event may have done. + * Having `DesiredCapacity` unset ensures the service remains at-capacity at all times. + * + * It'll also make the `MinInstancesInService` property dynamic via a CFN Parameter that Riff-Raff will set. + * The value depends on the current capacity of the ASG: + * - If the service is running normally, it'll be set to the `MinSize` capacity + * - If the service is partially scaled, it'll be set to the current `DesiredCapacity` + * - If the service is fully scaled, it'll be set to (at least) `MaxSize` - 1 + * + * @privateRemarks + * - Temporarily implemented as a singleton to ensure only one instance is added to a stack, + * else multiple attempts to add the same CFN Parameter will be made and fail. + * - Once out of experimental, instantiate this `Aspect` directly in {@link GuStack}. + * + * @see https://github.com/guardian/testing-asg-rolling-update + */ class HorizontallyScalingDeploymentProperties implements IAspect { public readonly stack: Stack; private static instance: HorizontallyScalingDeploymentProperties | undefined; @@ -49,16 +69,6 @@ class HorizontallyScalingDeploymentProperties implements IAspect { */ cfnAutoScalingGroup.desiredCapacity = undefined; - /* - An autoscaling group that horizontally scales should expose a CloudFormation Parameter linked to the - `MinInstancesInService` property of the rolling update policy. - - Riff-Raff will set this parameter during deployment. - The value depends on the current capacity of the ASG: - - If the service is running normally, it'll be set to the `Minimum` capacity - - If the service is partially scaled, it'll be set to the current `Desired` capacity - - If the service is fully scaled, it'll be set to (at least) `Maximum` - 1 - */ const minInstancesInService = new CfnParameter(guStack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { type: "Number", default: parseInt(cfnAutoScalingGroup.minSize), @@ -76,6 +86,12 @@ class HorizontallyScalingDeploymentProperties implements IAspect { } } +/** + * An IAM Policy allowing the sending of a CloudFormation signal. + * + * @privateRemarks + * Implemented as a singleton as the resources can only be tightened at most to the CloudFormation stack. + */ class AsgRollingUpdatePolicy extends Policy { private static instance: AsgRollingUpdatePolicy | undefined; @@ -118,32 +134,37 @@ export interface GuEc2AppExperimentalProps extends Omit Date: Fri, 13 Sep 2024 19:19:02 +0100 Subject: [PATCH 12/23] feat(experimental-ec2-pattern): Support an ASG w/multiple scaling policies --- src/experimental/patterns/ec2-app.test.ts | 80 +++++++++++++++++++++++ src/experimental/patterns/ec2-app.ts | 21 ++++-- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 5eae6792bf..aba7a91aa1 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -1,5 +1,6 @@ import { App, Duration } from "aws-cdk-lib"; import { Match, Template } from "aws-cdk-lib/assertions"; +import { CfnScalingPolicy } from "aws-cdk-lib/aws-autoscaling"; import { InstanceClass, InstanceSize, InstanceType, UserData } from "aws-cdk-lib/aws-ec2"; import { CloudFormationStackArtifact } from "aws-cdk-lib/cx-api"; import { AccessScope } from "../../constants"; @@ -296,4 +297,83 @@ describe("The GuEc2AppExperimental pattern", () => { }, }); }); + + it("should add a single CFN Parameter per ASG regardless of how many scaling policies are attached to it", () => { + const cdkApp = new App(); + const stack = new GuStack(cdkApp, "test", { + stack: "test-stack", + stage: "TEST", + }); + + const scalingApp = "my-scaling-app"; + const { autoScalingGroup } = new GuEc2AppExperimental(stack, { + ...initialProps(stack, scalingApp), + scaling: { + minimumInstances: 5, + }, + }); + autoScalingGroup.scaleOnRequestCount("ScaleOnRequests", { + targetRequestsPerMinute: 100, + }); + + new CfnScalingPolicy(autoScalingGroup, "ScaleOut", { + autoScalingGroupName: autoScalingGroup.autoScalingGroupName, + policyType: "SimpleScaling", + adjustmentType: "ChangeInCapacity", + scalingAdjustment: 1, + }); + + new CfnScalingPolicy(autoScalingGroup, "ScaleIn", { + autoScalingGroupName: autoScalingGroup.autoScalingGroupName, + policyType: "SimpleScaling", + adjustmentType: "ChangeInCapacity", + scalingAdjustment: -1, + }); + + /* + We're ultimately testing an `Aspect`, which appear to run only at synth time. + As a work-around, synth the `App`, then perform assertions on the resulting template. + + See also: https://github.com/aws/aws-cdk/issues/29047. + */ + const { artifacts } = cdkApp.synth(); + const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); + + if (!cfnStack) { + throw new Error("Unable to locate a CloudFormationStackArtifact"); + } + + const template = Template.fromJSON(cfnStack.template as Record); + + const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; + + template.hasParameter(parameterName, { + Type: "Number", + Default: 5, + MaxValue: 9, // (min * 2) - 1 + }); + + template.hasResource("AWS::AutoScaling::AutoScalingGroup", { + Properties: { + MinSize: "5", + MaxSize: "10", + DesiredCapacity: Match.absent(), + Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), + }, + UpdatePolicy: { + AutoScalingRollingUpdate: { + MaxBatchSize: 10, + SuspendProcesses: ["AlarmNotification"], + MinSuccessfulInstancesPercent: 100, + WaitOnResourceSignals: true, + PauseTime: "PT5M", + MinInstancesInService: { + Ref: parameterName, + }, + }, + }, + }); + + template.resourceCountIs("AWS::AutoScaling::ScalingPolicy", 3); + }); }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 203f4b7fdd..51f3047882 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -32,10 +32,12 @@ import { isSingletonPresentInStack } from "../../utils/singleton"; */ class HorizontallyScalingDeploymentProperties implements IAspect { public readonly stack: Stack; + private readonly asgToParamMap: Map; private static instance: HorizontallyScalingDeploymentProperties | undefined; private constructor(scope: GuStack) { this.stack = scope; + this.asgToParamMap = new Map(); } public static getInstance(stack: GuStack): HorizontallyScalingDeploymentProperties { @@ -69,11 +71,20 @@ class HorizontallyScalingDeploymentProperties implements IAspect { */ cfnAutoScalingGroup.desiredCapacity = undefined; - const minInstancesInService = new CfnParameter(guStack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { - type: "Number", - default: parseInt(cfnAutoScalingGroup.minSize), - maxValue: parseInt(cfnAutoScalingGroup.maxSize) - 1, - }); + const asgNodeId = autoScalingGroup.node.id; + + if (!this.asgToParamMap.has(asgNodeId)) { + this.asgToParamMap.set( + asgNodeId, + new CfnParameter(guStack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { + type: "Number", + default: parseInt(cfnAutoScalingGroup.minSize), + maxValue: parseInt(cfnAutoScalingGroup.maxSize) - 1, + }), + ); + } + + const minInstancesInService = this.asgToParamMap.get(asgNodeId)!; cfnAutoScalingGroup.cfnOptions.updatePolicy = { autoScalingRollingUpdate: { From 767ec0fca1c2049de9f2128b0706a72d18bef40a Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 20:03:16 +0100 Subject: [PATCH 13/23] docs(experimental-ec2-pattern): Document thrown error --- src/experimental/patterns/ec2-app.test.ts | 28 +++++++++++++++++++++++ src/experimental/patterns/ec2-app.ts | 18 ++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index aba7a91aa1..30f36cca70 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -376,4 +376,32 @@ describe("The GuEc2AppExperimental pattern", () => { template.resourceCountIs("AWS::AutoScaling::ScalingPolicy", 3); }); + + it("should throw an error when a scaling policy is not created with aa GuAutoScalingGroup scope", () => { + const cdkApp = new App(); + const stack = new GuStack(cdkApp, "test", { + stack: "test-stack", + stage: "TEST", + }); + + const { autoScalingGroup } = new GuEc2AppExperimental(stack, initialProps(stack)); + + /* + Should be created like this to avoid the error: + + new CfnScalingPolicy(autoScalingGroup, "ScaleOut", { ... }); + */ + new CfnScalingPolicy(stack, "ScaleOut", { + autoScalingGroupName: autoScalingGroup.autoScalingGroupName, + policyType: "SimpleScaling", + adjustmentType: "ChangeInCapacity", + scalingAdjustment: 1, + }); + + expect(() => { + cdkApp.synth(); + }).toThrowError( + "Failed to detect the autoscaling group relating to the scaling policy on path test/ScaleOut. Was it created in the scope of a GuAutoScalingGroup?", + ); + }); }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 51f3047882..fa34b16111 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -54,10 +54,26 @@ class HorizontallyScalingDeploymentProperties implements IAspect { const { scopes, path } = node; const guStack = GuStack.of(construct); + /* + Requiring a `CfnScalingPolicy` to be created in the scope of a `GuAutoScalingGroup` + is the most reliable way to associate the two together. + + Though the `autoScalingGroupName` property is passed to a `CfnScalingPolicy` when instantiating, + this does not create a concrete link as AWS CDK sets the value to be the ASG's `Ref`. + That is, it is a `Token` until it is synthesised. + This is even if the ASG has an explicit name set. + + See also: + - https://docs.aws.amazon.com/cdk/v2/guide/tokens.html + - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-autoscaling-autoscalinggroup.html#aws-resource-autoscaling-autoscalinggroup-return-values + - https://github.com/aws/aws-cdk/blob/f6b649d47f8bc30ca741fbb7a4852d51e8275002/packages/aws-cdk-lib/aws-autoscaling/lib/auto-scaling-group.ts#L1560 + */ const autoScalingGroup = scopes.find((_): _ is GuAutoScalingGroup => _ instanceof GuAutoScalingGroup); if (!autoScalingGroup) { - throw new Error(`Failed to detect the autoscaling group relating to the scaling policy on path ${path}`); + throw new Error( + `Failed to detect the autoscaling group relating to the scaling policy on path ${path}. Was it created in the scope of a GuAutoScalingGroup?`, + ); } const cfnAutoScalingGroup = autoScalingGroup.node.defaultChild as CfnAutoScalingGroup; From b8f6ff59beb943426abbf8dea37b97422770726b Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 13 Sep 2024 20:39:48 +0100 Subject: [PATCH 14/23] refactor(experimental-ec2-pattern): Simplify by reusing variable --- src/experimental/patterns/ec2-app.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index fa34b16111..56e1a61ea5 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -1,11 +1,11 @@ -import type { IAspect, Stack } from "aws-cdk-lib"; +import type { IAspect } from "aws-cdk-lib"; import { Aspects, CfnParameter, Duration } from "aws-cdk-lib"; import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; import { CfnScalingPolicy, ScalingProcess, UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; import { Effect, Policy, PolicyStatement } from "aws-cdk-lib/aws-iam"; import type { IConstruct } from "constructs"; import { GuAutoScalingGroup } from "../../constructs/autoscaling"; -import { GuStack } from "../../constructs/core"; +import type { GuStack } from "../../constructs/core"; import type { GuEc2AppProps } from "../../patterns"; import { GuEc2App } from "../../patterns"; import { isSingletonPresentInStack } from "../../utils/singleton"; @@ -31,7 +31,7 @@ import { isSingletonPresentInStack } from "../../utils/singleton"; * @see https://github.com/guardian/testing-asg-rolling-update */ class HorizontallyScalingDeploymentProperties implements IAspect { - public readonly stack: Stack; + public readonly stack: GuStack; private readonly asgToParamMap: Map; private static instance: HorizontallyScalingDeploymentProperties | undefined; @@ -52,7 +52,6 @@ class HorizontallyScalingDeploymentProperties implements IAspect { if (construct instanceof CfnScalingPolicy) { const { node } = construct; const { scopes, path } = node; - const guStack = GuStack.of(construct); /* Requiring a `CfnScalingPolicy` to be created in the scope of a `GuAutoScalingGroup` @@ -92,7 +91,7 @@ class HorizontallyScalingDeploymentProperties implements IAspect { if (!this.asgToParamMap.has(asgNodeId)) { this.asgToParamMap.set( asgNodeId, - new CfnParameter(guStack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { + new CfnParameter(this.stack, `MinInstancesInServiceFor${autoScalingGroup.app}`, { type: "Number", default: parseInt(cfnAutoScalingGroup.minSize), maxValue: parseInt(cfnAutoScalingGroup.maxSize) - 1, From b0718b4356c5dfb0bee474e6a2bb1b3ca4ae1cc8 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Sat, 14 Sep 2024 14:25:03 +0100 Subject: [PATCH 15/23] fix(experimental-ec2-pattern): Obtain instance id more reliably The `ec2metadata` command was failing with a 401 with AMIable CODE in deployTools account: ```console root@ip-10-248-51-213:/var/lib/cloud/instance# ec2metadata --instance-id Traceback (most recent call last): File "/usr/bin/ec2metadata", line 249, in main() File "/usr/bin/ec2metadata", line 245, in main display(metaopts, burl, prefix) File "/usr/bin/ec2metadata", line 192, in display value = m.get(metaopt) File "/usr/bin/ec2metadata", line 177, in get return self._get('meta-data/' + metaopt) File "/usr/bin/ec2metadata", line 137, in _get resp = urllib_request.urlopen(urllib_request.Request(url)) File "/usr/lib/python3.8/urllib/request.py", line 222, in urlopen return opener.open(url, data, timeout) File "/usr/lib/python3.8/urllib/request.py", line 531, in open response = meth(req, response) File "/usr/lib/python3.8/urllib/request.py", line 640, in http_response response = self.parent.error( File "/usr/lib/python3.8/urllib/request.py", line 569, in error return self._call_chain(*args) File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain result = func(*args) File "/usr/lib/python3.8/urllib/request.py", line 649, in http_error_default raise HTTPError(req.full_url, code, msg, hdrs, fp) urllib.error.HTTPError: HTTP Error 401: Unautho ``` This service uses IMDSv2. A 401 response usually happens when a request is made without a token. However `ec2metadata` does exchange a token. Switch to a more reliable mechanism. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html. --- .../patterns/__snapshots__/ec2-app.test.ts.snap | 3 ++- src/experimental/patterns/ec2-app.ts | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index 1f383c074f..8da456837b 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -920,7 +920,8 @@ aws s3 cp 's3://", dpkg -i /test-gu-ec2-app/test-gu-ec2-app-123.deb # GuEc2AppExperimental UserData Start - INSTANCE_ID=$(ec2metadata --instance-id) + TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/instance-id") STATE=$(aws elbv2 describe-target-health --target-group-arn ", { diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 56e1a61ea5..8243938808 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -251,16 +251,14 @@ export class GuEc2AppExperimental extends GuEc2App { AsgRollingUpdatePolicy.getInstance(scope).attachToRole(role); /* - `ec2metadata` is available via `cloud-utils` installed on all Canonical Ubuntu AMIs. - See https://github.com/canonical/cloud-utils. - `aws` is available via AMIgo baked AMIs. See https://github.com/guardian/amigo/tree/main/roles/aws-tools. */ userData.addCommands( `# ${GuEc2AppExperimental.name} UserData Start`, ` - INSTANCE_ID=$(ec2metadata --instance-id) + TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/instance-id") STATE=$(aws elbv2 describe-target-health \ --target-group-arn ${targetGroup.targetGroupArn} \ From f4e2a7c176eb4d793c4b943afaa1707bf237e6e2 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Mon, 16 Sep 2024 10:28:34 +0100 Subject: [PATCH 16/23] docs(experimental-ec2-pattern): Add changeset --- .changeset/happy-badgers-compare.md | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .changeset/happy-badgers-compare.md diff --git a/.changeset/happy-badgers-compare.md b/.changeset/happy-badgers-compare.md new file mode 100644 index 0000000000..4596a10a93 --- /dev/null +++ b/.changeset/happy-badgers-compare.md @@ -0,0 +1,43 @@ +--- +"@guardian/cdk": minor +--- + +feat(experimental-ec2-pattern): Pattern to deploy ASG updates w/CFN + +Included in this update is a new experimental pattern `GuEc2AppExperimental`, which can be used in place of a `GuEc2App`: + +```ts +import { GuEc2AppExperimental } from "@guardian/cdk/lib/experimental/patterns/ec2-app"; +``` + +This pattern will add an [`AutoScalingRollingUpdate` policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-attribute-updatepolicy.html#cfn-attributes-updatepolicy-rollingupdate) +to the autoscaling group. +This allows application updates to be performed like a standard CloudFormation update, +and using the custom logic provided by Riff-Raff's `autoscaling` deployment type is unnecessary. + +This experimental pattern has few requirements. + +## Add the build number to the application artifact +This change requires versioned artifacts. + +The easiest way to achieve this is by adding the build number to the filename of the artifact: + +```ts +import { UserData } from "aws-cdk-lib/aws-ec2"; +// Use a GitHub Actions provided environment variable +const buildNumber = process.env.GITHUB_RUN_NUMBER ?? "DEV"; + +const userData = UserData.forLinux(); +userData.addCommands(`aws s3 cp s3://dist-bucket/path/to/artifact-${buildNumber}.deb /tmp/artifact.deb`); +userData.addCommands(`dpkg -i /tmp/artifact.dep`); +``` + +## `riff-raff.yaml` +The `riff-raff.yaml` file should remove the `deploy` action of the `autoscaling` deployment type. +Though including it shouldn't break anything, it would result in a longer deployment time as instance will be rotated by both CloudFormation and Riff-Raff's custom logic. + +The `uploadArtifacts` step of the `autoscaling` deployment type should still be included, with the `cloud-formation` deployment type depending on it. +This step uploads the versioned artifact to S3. + +> [!TIP] +> An [auto-generated `riff-raff.yaml` file](https://github.com/guardian/cdk/blob/main/src/riff-raff-yaml-file/README.md) meets this requirement. From 98f8f9fb4cf9f867c0d3345feab6476cfec8e966 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 07:41:50 +0100 Subject: [PATCH 17/23] refactor(experimental-ec2-pattern): Simplify tests by checking minimal properties --- src/experimental/patterns/ec2-app.test.ts | 34 ++--------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 30f36cca70..4d06ce58cd 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -185,18 +185,10 @@ describe("The GuEc2AppExperimental pattern", () => { template.hasResource("AWS::AutoScaling::AutoScalingGroup", { Properties: { - MinSize: "5", - MaxSize: "10", DesiredCapacity: Match.absent(), - Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), }, UpdatePolicy: { AutoScalingRollingUpdate: { - MaxBatchSize: 10, - SuspendProcesses: ["AlarmNotification"], - MinSuccessfulInstancesPercent: 100, - WaitOnResourceSignals: true, - PauseTime: "PT5M", MinInstancesInService: { Ref: parameterName, }, @@ -241,11 +233,7 @@ describe("The GuEc2AppExperimental pattern", () => { const template = Template.fromJSON(cfnStack.template as Record); - /* - The scaling ASG should: - - Not have `DesiredCapacity` set - - Have `MinInstancesInService` set via a CFN Parameter - */ + // The scaling ASG should NOT have `DesiredCapacity` set, and `MinInstancesInService` set via a CFN Parameter const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; template.hasParameter(parameterName, { Type: "Number", @@ -254,18 +242,11 @@ describe("The GuEc2AppExperimental pattern", () => { }); template.hasResource("AWS::AutoScaling::AutoScalingGroup", { Properties: { - MinSize: "5", - MaxSize: "10", DesiredCapacity: Match.absent(), Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), }, UpdatePolicy: { AutoScalingRollingUpdate: { - MaxBatchSize: 10, - SuspendProcesses: ["AlarmNotification"], - MinSuccessfulInstancesPercent: 100, - WaitOnResourceSignals: true, - PauseTime: "PT5M", MinInstancesInService: { Ref: parameterName, }, @@ -273,25 +254,14 @@ describe("The GuEc2AppExperimental pattern", () => { }, }); - /* - The static ASG should: - - Have `DesiredCapacity` set explicitly - - Have `MinInstancesInService` set explicitly - */ + // The static ASG SHOULD have `DesiredCapacity` and `MinInstancesInService` explicitly template.hasResource("AWS::AutoScaling::AutoScalingGroup", { Properties: { - MinSize: "1", - MaxSize: "2", DesiredCapacity: "1", Tags: Match.arrayWith([{ Key: "App", Value: staticApp, PropagateAtLaunch: true }]), }, UpdatePolicy: { AutoScalingRollingUpdate: { - MaxBatchSize: 2, - SuspendProcesses: ["AlarmNotification"], - MinSuccessfulInstancesPercent: 100, - WaitOnResourceSignals: true, - PauseTime: "PT5M", MinInstancesInService: 1, }, }, From 37c9533e82973ee09e8ddf6075472576a9f662e6 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 07:45:50 +0100 Subject: [PATCH 18/23] refactor(experimental-ec2-pattern): Remove duplicated test The behaviour being tested is already covered by `should only adjust properties of a horizontally scaling service`. --- src/experimental/patterns/ec2-app.test.ts | 55 ----------------------- 1 file changed, 55 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 4d06ce58cd..b0c56ae07f 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -142,61 +142,6 @@ describe("The GuEc2AppExperimental pattern", () => { expect(endMarkerPosition).toEqual(totalLines - 1); }); - it("should adjust properties of a horizontally scaling service", () => { - const cdkApp = new App(); - const stack = new GuStack(cdkApp, "test", { - stack: "test-stack", - stage: "TEST", - }); - - const scalingApp = "my-scaling-app"; - const { autoScalingGroup } = new GuEc2AppExperimental(stack, { - ...initialProps(stack, scalingApp), - scaling: { - minimumInstances: 5, - }, - }); - autoScalingGroup.scaleOnRequestCount("ScaleOnRequests", { - targetRequestsPerMinute: 100, - }); - - /* - We're ultimately testing an `Aspect`, which appear to run only at synth time. - As a work-around, synth the `App`, then perform assertions on the resulting template. - - See also: https://github.com/aws/aws-cdk/issues/29047. - */ - const { artifacts } = cdkApp.synth(); - const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); - - if (!cfnStack) { - throw new Error("Unable to locate a CloudFormationStackArtifact"); - } - - const template = Template.fromJSON(cfnStack.template as Record); - - const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; - - template.hasParameter(parameterName, { - Type: "Number", - Default: 5, - MaxValue: 9, // (min * 2) - 1 - }); - - template.hasResource("AWS::AutoScaling::AutoScalingGroup", { - Properties: { - DesiredCapacity: Match.absent(), - }, - UpdatePolicy: { - AutoScalingRollingUpdate: { - MinInstancesInService: { - Ref: parameterName, - }, - }, - }, - }); - }); - it("should only adjust properties of a horizontally scaling service", () => { const cdkApp = new App(); const stack = new GuStack(cdkApp, "test", { From a259531e7d90f76a4b74577326580e43f03b9fc3 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 07:47:34 +0100 Subject: [PATCH 19/23] refactor(experimental-ec2-pattern): Wrap code example comment in doc string --- src/experimental/patterns/ec2-app.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index b0c56ae07f..e5a4daefdc 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -292,7 +292,7 @@ describe("The GuEc2AppExperimental pattern", () => { template.resourceCountIs("AWS::AutoScaling::ScalingPolicy", 3); }); - it("should throw an error when a scaling policy is not created with aa GuAutoScalingGroup scope", () => { + it("should throw an error when a scaling policy is not created with a GuAutoScalingGroup scope", () => { const cdkApp = new App(); const stack = new GuStack(cdkApp, "test", { stack: "test-stack", @@ -301,10 +301,14 @@ describe("The GuEc2AppExperimental pattern", () => { const { autoScalingGroup } = new GuEc2AppExperimental(stack, initialProps(stack)); - /* - Should be created like this to avoid the error: - - new CfnScalingPolicy(autoScalingGroup, "ScaleOut", { ... }); + /** + * Should be created like this to avoid the error: + * + * @example + * ```ts + * declare const autoScalingGroup: GuAutoScalingGroup; + * new CfnScalingPolicy(autoScalingGroup, "ScaleOut", { ... }); + * ``` */ new CfnScalingPolicy(stack, "ScaleOut", { autoScalingGroupName: autoScalingGroup.autoScalingGroupName, From 7b330b6231bbc81d7c2dac379f3285b2bfefee12 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 08:46:00 +0100 Subject: [PATCH 20/23] refactor(riff-raff.yaml): Simplify handling of ASG update policy --- src/riff-raff-yaml-file/index.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/riff-raff-yaml-file/index.ts b/src/riff-raff-yaml-file/index.ts index c33c453bb7..539ded7d5d 100644 --- a/src/riff-raff-yaml-file/index.ts +++ b/src/riff-raff-yaml-file/index.ts @@ -254,15 +254,14 @@ export class RiffRaffYamlFile { deployments.set(lambdaDeployment.name, lambdaDeployment.props); }); - // ASGs without an UpdatePolicy can be deployed via Riff-Raff's (legacy) `autoscaling` deployment type. - // ASGs with an UpdatePolicy are updated via Riff-Raff's `cloud-formation` deployment type. + /* + Instances in an ASG with an `AutoScalingRollingUpdate` update policy are rotated via CloudFormation. + Therefore, they do not need to also perform an `autoscaling` deployment via Riff-Raff. + */ const legacyAutoscalingGroups = autoscalingGroups.filter((asg) => { const { cfnOptions } = asg.node.defaultChild as CfnAutoScalingGroup; const { updatePolicy } = cfnOptions; - return ( - updatePolicy?.autoScalingReplacingUpdate === undefined && - updatePolicy?.autoScalingRollingUpdate === undefined - ); + return updatePolicy?.autoScalingRollingUpdate === undefined; }); legacyAutoscalingGroups.forEach((asg) => { From 3ebd3436065a40a78432c07b953f191bc3ac16cd Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 12:53:05 +0100 Subject: [PATCH 21/23] refactor: Extract repeated logic into function --- src/experimental/patterns/ec2-app.test.ts | 67 ++++++++++------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index e5a4daefdc..0c32d949db 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -10,6 +10,31 @@ import { simpleGuStackForTesting } from "../../utils/test"; import type { GuEc2AppExperimentalProps } from "./ec2-app"; import { GuEc2AppExperimental } from "./ec2-app"; +/** + * `Aspects` appear to run only at synth time. + * This means we must synth the stack to see the results of the `Aspect`. + * + * @see https://github.com/aws/aws-cdk/issues/29047 + * + * @param stack the stack to synthesise + */ +function getTemplateAfterAspectInvocation(stack: GuStack): Template { + const app = App.of(stack); + + if (!app) { + throw new Error(`Unable to locate the enclosing App from GuStack ${stack.node.id}`); + } + + const { artifacts } = app.synth(); + const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); + + if (!cfnStack) { + throw new Error("Unable to locate a CloudFormationStackArtifact"); + } + + return Template.fromJSON(cfnStack.template as Record); +} + // TODO test User Data includes a build number describe("The GuEc2AppExperimental pattern", () => { function initialProps(scope: GuStack, app: string = "test-gu-ec2-app"): GuEc2AppExperimentalProps { @@ -143,11 +168,7 @@ describe("The GuEc2AppExperimental pattern", () => { }); it("should only adjust properties of a horizontally scaling service", () => { - const cdkApp = new App(); - const stack = new GuStack(cdkApp, "test", { - stack: "test-stack", - stage: "TEST", - }); + const stack = simpleGuStackForTesting(); const scalingApp = "my-scaling-app"; const { autoScalingGroup } = new GuEc2AppExperimental(stack, { @@ -163,20 +184,7 @@ describe("The GuEc2AppExperimental pattern", () => { const staticApp = "my-static-app"; new GuEc2AppExperimental(stack, initialProps(stack, staticApp)); - /* - We're ultimately testing an `Aspect`, which appear to run only at synth time. - As a work-around, synth the `App`, then perform assertions on the resulting template. - - See also: https://github.com/aws/aws-cdk/issues/29047. - */ - const { artifacts } = cdkApp.synth(); - const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); - - if (!cfnStack) { - throw new Error("Unable to locate a CloudFormationStackArtifact"); - } - - const template = Template.fromJSON(cfnStack.template as Record); + const template = getTemplateAfterAspectInvocation(stack); // The scaling ASG should NOT have `DesiredCapacity` set, and `MinInstancesInService` set via a CFN Parameter const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; @@ -214,11 +222,7 @@ describe("The GuEc2AppExperimental pattern", () => { }); it("should add a single CFN Parameter per ASG regardless of how many scaling policies are attached to it", () => { - const cdkApp = new App(); - const stack = new GuStack(cdkApp, "test", { - stack: "test-stack", - stage: "TEST", - }); + const stack = simpleGuStackForTesting(); const scalingApp = "my-scaling-app"; const { autoScalingGroup } = new GuEc2AppExperimental(stack, { @@ -245,20 +249,7 @@ describe("The GuEc2AppExperimental pattern", () => { scalingAdjustment: -1, }); - /* - We're ultimately testing an `Aspect`, which appear to run only at synth time. - As a work-around, synth the `App`, then perform assertions on the resulting template. - - See also: https://github.com/aws/aws-cdk/issues/29047. - */ - const { artifacts } = cdkApp.synth(); - const cfnStack = artifacts.find((_): _ is CloudFormationStackArtifact => _ instanceof CloudFormationStackArtifact); - - if (!cfnStack) { - throw new Error("Unable to locate a CloudFormationStackArtifact"); - } - - const template = Template.fromJSON(cfnStack.template as Record); + const template = getTemplateAfterAspectInvocation(stack); const parameterName = `MinInstancesInServiceFor${scalingApp.replaceAll("-", "")}`; From 4d96a1a2f175ce370bbd6e165d1e416c21d2166a Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 13:05:46 +0100 Subject: [PATCH 22/23] refactor(experimental-ec2-pattern): Simplify tests by checking minimal properties --- src/experimental/patterns/ec2-app.test.ts | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 0c32d949db..79a4411116 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -94,12 +94,8 @@ describe("The GuEc2AppExperimental pattern", () => { MinSize: "5", }, CreationPolicy: { - AutoScalingCreationPolicy: { - MinSuccessfulInstancesPercent: 100, - }, ResourceSignal: { Count: 5, - Timeout: "PT5M", }, }, }); @@ -259,20 +255,10 @@ describe("The GuEc2AppExperimental pattern", () => { MaxValue: 9, // (min * 2) - 1 }); + template.resourceCountIs("AWS::AutoScaling::AutoScalingGroup", 1); template.hasResource("AWS::AutoScaling::AutoScalingGroup", { - Properties: { - MinSize: "5", - MaxSize: "10", - DesiredCapacity: Match.absent(), - Tags: Match.arrayWith([{ Key: "App", Value: scalingApp, PropagateAtLaunch: true }]), - }, UpdatePolicy: { AutoScalingRollingUpdate: { - MaxBatchSize: 10, - SuspendProcesses: ["AlarmNotification"], - MinSuccessfulInstancesPercent: 100, - WaitOnResourceSignals: true, - PauseTime: "PT5M", MinInstancesInService: { Ref: parameterName, }, From 7a12f608297ced73a44c50b73c7312e22ffeafef Mon Sep 17 00:00:00 2001 From: akash1810 Date: Tue, 17 Sep 2024 14:18:07 +0100 Subject: [PATCH 23/23] fix(experimental-ec2-pattern): Set `PauseTime` from healthcheck grace period Matching these properties allows rollbacks to happen as quickly as possible. --- .../__snapshots__/ec2-app.test.ts.snap | 4 +- src/experimental/patterns/ec2-app.test.ts | 37 +++----- src/experimental/patterns/ec2-app.ts | 85 ++++++++++++++----- 3 files changed, 81 insertions(+), 45 deletions(-) diff --git a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap index 8da456837b..11b0ea2800 100644 --- a/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap +++ b/src/experimental/patterns/__snapshots__/ec2-app.test.ts.snap @@ -106,7 +106,7 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` }, "ResourceSignal": { "Count": 1, - "Timeout": "PT5M", + "Timeout": "PT2M", }, }, "Properties": { @@ -180,7 +180,7 @@ exports[`The GuEc2AppExperimental pattern matches the snapshot 1`] = ` "MaxBatchSize": 2, "MinInstancesInService": 1, "MinSuccessfulInstancesPercent": 100, - "PauseTime": "PT5M", + "PauseTime": "PT2M", "SuspendProcesses": [ "AlarmNotification", ], diff --git a/src/experimental/patterns/ec2-app.test.ts b/src/experimental/patterns/ec2-app.test.ts index 79a4411116..a65857ef32 100644 --- a/src/experimental/patterns/ec2-app.test.ts +++ b/src/experimental/patterns/ec2-app.test.ts @@ -1,5 +1,6 @@ import { App, Duration } from "aws-cdk-lib"; import { Match, Template } from "aws-cdk-lib/assertions"; +import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; import { CfnScalingPolicy } from "aws-cdk-lib/aws-autoscaling"; import { InstanceClass, InstanceSize, InstanceType, UserData } from "aws-cdk-lib/aws-ec2"; import { CloudFormationStackArtifact } from "aws-cdk-lib/cx-api"; @@ -101,39 +102,29 @@ describe("The GuEc2AppExperimental pattern", () => { }); }); - it("should create an ASG with the maximum resource signal timeout", () => { + it("should have a PauseTime equal to the ASG healthcheck grace period", () => { const stack = simpleGuStackForTesting(); + const { autoScalingGroup } = new GuEc2AppExperimental(stack, initialProps(stack)); - const targetGroupHealthcheckTimeout = Duration.minutes(7); - - new GuEc2AppExperimental(stack, { - ...initialProps(stack), - healthcheck: { - timeout: targetGroupHealthcheckTimeout, - interval: Duration.seconds(targetGroupHealthcheckTimeout.toSeconds() + 60), - }, - }); + const tenMinutes = Duration.minutes(10); - const template = Template.fromStack(stack); + const cfnAsg = autoScalingGroup.node.defaultChild as CfnAutoScalingGroup; + cfnAsg.healthCheckGracePeriod = tenMinutes.toSeconds(); - // The Target Group times out in 7 minutes. - template.hasResourceProperties("AWS::ElasticLoadBalancingV2::TargetGroup", { - HealthCheckTimeoutSeconds: targetGroupHealthcheckTimeout.toSeconds(), - }); + const template = getTemplateAfterAspectInvocation(stack); - // The ASG grace period is 2 minutes, which is less than the Target Group. - // Therefore, the resource signal timeout should be 7 minutes. template.hasResource("AWS::AutoScaling::AutoScalingGroup", { Properties: { - HealthCheckGracePeriod: 120, + HealthCheckGracePeriod: tenMinutes.toSeconds(), }, CreationPolicy: { - AutoScalingCreationPolicy: { - MinSuccessfulInstancesPercent: 100, - }, ResourceSignal: { - Count: 1, - Timeout: targetGroupHealthcheckTimeout.toIsoString(), + Timeout: tenMinutes.toIsoString(), + }, + }, + UpdatePolicy: { + AutoScalingRollingUpdate: { + PauseTime: tenMinutes.toIsoString(), }, }, }); diff --git a/src/experimental/patterns/ec2-app.ts b/src/experimental/patterns/ec2-app.ts index 8243938808..cf1b1aad1c 100644 --- a/src/experimental/patterns/ec2-app.ts +++ b/src/experimental/patterns/ec2-app.ts @@ -1,7 +1,6 @@ import type { IAspect } from "aws-cdk-lib"; import { Aspects, CfnParameter, Duration } from "aws-cdk-lib"; -import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; -import { CfnScalingPolicy, ScalingProcess, UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; +import { CfnAutoScalingGroup, CfnScalingPolicy, ScalingProcess, UpdatePolicy } from "aws-cdk-lib/aws-autoscaling"; import { Effect, Policy, PolicyStatement } from "aws-cdk-lib/aws-iam"; import type { IConstruct } from "constructs"; import { GuAutoScalingGroup } from "../../constructs/autoscaling"; @@ -10,6 +9,68 @@ import type { GuEc2AppProps } from "../../patterns"; import { GuEc2App } from "../../patterns"; import { isSingletonPresentInStack } from "../../utils/singleton"; +/** + * Ensures the `AutoScalingRollingUpdate` of an AutoScaling Group has a `PauseTime` matching the healthcheck grace period. + * It also ensures the `CreationPolicy` resource signal `Timeout` matches the healthcheck grace period. + * + * @privateRemarks + * The ASG healthcheck grace period is hard-coded by {@link GuEc2App}. + * Customisation of this value is performed via an escape hatch. + * An `Aspect` is the only way to observe any customisation. + * + * TODO Expose the healthcheck grace period as a property on {@link GuEc2App} and remove this `Aspect`. + */ +class AutoScalingRollingUpdateTimeout implements IAspect { + public readonly stack: GuStack; + private static instance: AutoScalingRollingUpdateTimeout | undefined; + + private constructor(scope: GuStack) { + this.stack = scope; + } + + public static getInstance(stack: GuStack): AutoScalingRollingUpdateTimeout { + if (!this.instance || !isSingletonPresentInStack(stack, this.instance)) { + this.instance = new AutoScalingRollingUpdateTimeout(stack); + } + return this.instance; + } + + public visit(construct: IConstruct) { + if (construct instanceof CfnAutoScalingGroup) { + const currentRollingUpdate = construct.cfnOptions.updatePolicy?.autoScalingRollingUpdate; + const currentCreationPolicy = construct.cfnOptions.creationPolicy; + + /** + * The type of `healthCheckGracePeriod` is `number | undefined`. + * In reality, it will always be set as we set it in {@link GuEc2App}. + * The right-hand side to appease the compiler; 5 minutes is the default value used by AWS. + * + * @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-attribute-updatepolicy.html#cfn-attributes-updatepolicy-rollingupdate-pausetime + */ + const signalTimeoutSeconds = construct.healthCheckGracePeriod ?? Duration.minutes(5).toSeconds(); + + if (currentRollingUpdate) { + construct.cfnOptions.updatePolicy = { + autoScalingRollingUpdate: { + ...currentRollingUpdate, + pauseTime: Duration.seconds(signalTimeoutSeconds).toIsoString(), + }, + }; + } + + if (currentCreationPolicy) { + construct.cfnOptions.creationPolicy = { + ...currentCreationPolicy, + resourceSignal: { + ...currentCreationPolicy.resourceSignal, + timeout: Duration.seconds(signalTimeoutSeconds).toIsoString(), + }, + }; + } + } + } +} + /** * An `Aspect` that adjusts the properties of an AutoScaling Group using an `AutoScalingRollingUpdate` update policy. * @@ -222,29 +283,12 @@ export class GuEc2AppExperimental extends GuEc2App { cfnAutoScalingGroup.desiredCapacity = minimumInstances.toString(); - // TODO are these sensible values? - const signalTimeoutSeconds = Math.max( - targetGroup.healthCheck.timeout?.toSeconds() ?? 0, - cfnAutoScalingGroup.healthCheckGracePeriod ?? 0, - Duration.minutes(5).toSeconds(), - ); - - const currentRollingUpdate = cfnAutoScalingGroup.cfnOptions.updatePolicy?.autoScalingRollingUpdate; - - cfnAutoScalingGroup.cfnOptions.updatePolicy = { - autoScalingRollingUpdate: { - ...currentRollingUpdate, - pauseTime: Duration.seconds(signalTimeoutSeconds).toIsoString(), - }, - }; - cfnAutoScalingGroup.cfnOptions.creationPolicy = { autoScalingCreationPolicy: { minSuccessfulInstancesPercent: 100, }, resourceSignal: { count: minimumInstances, - timeout: Duration.seconds(signalTimeoutSeconds).toIsoString(), }, }; @@ -290,7 +334,8 @@ export class GuEc2AppExperimental extends GuEc2App { `, ); - // TODO Once out of experimental, instantiate this `Aspect` directly in `GuStack`. + // TODO Once out of experimental, instantiate these `Aspect`s directly in `GuStack`. + Aspects.of(scope).add(AutoScalingRollingUpdateTimeout.getInstance(scope)); Aspects.of(scope).add(HorizontallyScalingDeploymentProperties.getInstance(scope)); } }