feat: Perform EC2 deployments via CloudFormation

.github/workflows/ci-sbt.yml

@@ -38,19 +38,27 @@ jobs:
           cache: 'npm'
           cache-dependency-path: 'cdk/package-lock.json'
 
+      # This step creates an environment variable `BUILD_NUMBER`.
+      # It is used by:
+      #   - The `script/ci` script
+      #   - The CDK infrastructure
+      #   - The `guardian/actions-riff-raff` GitHub Action
+      - run: |
+          LAST_TEAMCITY_BUILD=1265
+          echo "BUILD_NUMBER=$(( $GITHUB_RUN_NUMBER + $LAST_TEAMCITY_BUILD ))" >> $GITHUB_ENV
+
       - name: Run script/ci
         run: ./script/ci
 
       - uses: guardian/actions-riff-raff@a8a8cabedb56d2d8922017efedba7fd09e5e6980 # v4.0.6
         with:
           projectName: security-hq
           roleArn: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }}
-          # Seed the build number with last number from TeamCity.
-          buildNumberOffset: 1265
+          buildNumber: ${{ env.BUILD_NUMBER }}
           githubToken: ${{ secrets.GITHUB_TOKEN }}
           configPath: hq/conf/riff-raff.yaml
           contentDirectories: |
             security-hq-cfn:
               - cdk/cdk.out/security-hq.template.json
             security-hq:
-              - hq/target/security-hq.deb
+              - dist

cdk/bin/cdk.ts

@@ -3,9 +3,11 @@ import { App } from "aws-cdk-lib";
 import { SecurityHQ } from "../lib/security-hq";
 
 const app = new App();
+
 new SecurityHQ(app, "security-hq", {
   stack: "security",
   stage: "PROD",
   cloudFormationStackName: "security-hq-PROD",
   env: { region: "eu-west-1" },
+  buildIdentifier: process.env.BUILD_NUMBER ?? "DEV"
 });

cdk/lib/__snapshots__/security-hq.test.ts.snap

@@ -18,7 +18,7 @@ exports[`HQ stack matches the snapshot 1`] = `
       "GuVpcParameter",
       "GuSubnetListParameter",
       "GuSubnetListParameter",
-      "GuEc2App",
+      "GuEc2AppExperimental",
       "GuCertificate",
       "GuInstanceRole",
       "GuSsmSshPolicy",
@@ -123,6 +123,34 @@ exports[`HQ stack matches the snapshot 1`] = `
       },
       "Type": "AWS::SSM::Parameter",
     },
+    "AsgRollingUpdatePolicy2A1DDC6F": {
+      "Properties": {
+        "PolicyDocument": {
+          "Statement": [
+            {
+              "Action": "cloudformation:SignalResource",
+              "Effect": "Allow",
+              "Resource": {
+                "Ref": "AWS::StackId",
+              },
+            },
+            {
+              "Action": "elasticloadbalancing:DescribeTargetHealth",
+              "Effect": "Allow",
+              "Resource": "*",
+            },
+          ],
+          "Version": "2012-10-17",
+        },
+        "PolicyName": "AsgRollingUpdatePolicy2A1DDC6F",
+        "Roles": [
+          {
+            "Ref": "InstanceRoleSecurityhq7C08CA33",
+          },
+        ],
+      },
+      "Type": "AWS::IAM::Policy",
+    },
     "AssumeRole3910C09D": {
       "Properties": {
         "PolicyDocument": {
@@ -145,7 +173,17 @@ exports[`HQ stack matches the snapshot 1`] = `
       "Type": "AWS::IAM::Policy",
     },
     "AutoScalingGroupSecurityhqASG8DD277A5": {
+      "CreationPolicy": {
+        "AutoScalingCreationPolicy": {
+          "MinSuccessfulInstancesPercent": 100,
+        },
+        "ResourceSignal": {
+          "Count": 1,
+          "Timeout": "PT2M",
+        },
+      },
       "Properties": {
+        "DesiredCapacity": "1",
         "HealthCheckGracePeriod": 120,
         "HealthCheckType": "ELB",
         "LaunchTemplate": {
@@ -163,10 +201,6 @@ exports[`HQ stack matches the snapshot 1`] = `
         "MetricsCollection": [
           {
             "Granularity": "1Minute",
-            "Metrics": [
-              "GroupTotalInstances",
-              "GroupInServiceInstances",
-            ],
           },
         ],
         "MinSize": "1",
@@ -219,6 +253,18 @@ exports[`HQ stack matches the snapshot 1`] = `
         },
       },
       "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "UpdatePolicy": {
+        "AutoScalingRollingUpdate": {
+          "MaxBatchSize": 2,
+          "MinInstancesInService": 1,
+          "MinSuccessfulInstancesPercent": 100,
+          "PauseTime": "PT2M",
+          "SuspendProcesses": [
+            "AlarmNotification",
+          ],
+          "WaitOnResourceSignals": true,
+        },
+      },
     },
     "CertificateSecurityhqD266EC9D": {
       "DeletionPolicy": "Retain",
@@ -1553,6 +1599,21 @@ exports[`HQ stack matches the snapshot 1`] = `
                 "",
                 [
                   "#!/bin/bash
+function exitTrap(){
+exitCode=$?
+
+        cfn-signal --stack ",
+                  {
+                    "Ref": "AWS::StackId",
+                  },
+                  "           --resource AutoScalingGroupSecurityhqASG8DD277A5           --region ",
+                  {
+                    "Ref": "AWS::Region",
+                  },
+                  "           --exit-code $exitCode || echo 'Failed to send Cloudformation Signal'
+
+}
+trap exitTrap EXIT
 # setup security-hq
     mkdir -p /etc/gu
 
@@ -1570,9 +1631,41 @@ exports[`HQ stack matches the snapshot 1`] = `
                   {
                     "Ref": "DistributionBucketName",
                   },
-                  "/security/PROD/security-hq/security-hq.deb /tmp/installer.deb
+                  "/security/PROD/security-hq/security-hq-TEST.deb /tmp/installer.deb
+
+    dpkg -i /tmp/installer.deb
+# GuEc2AppExperimental UserData Start
+
+      TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+      INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/instance-id")
+
+      STATE=$(aws elbv2 describe-target-health         --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupSecurityhq530DEDAA",
+                  },
+                  "         --region ",
+                  {
+                    "Ref": "AWS::Region",
+                  },
+                  "         --targets Id=$INSTANCE_ID,Port=9000         --query "TargetHealthDescriptions[0].TargetHealth.State")
+
+      until [ "$STATE" == "\\"healthy\\"" ]; do
+        echo "Instance not yet healthy within target group. Current state $STATE. Sleeping..."
+        sleep 5
+        STATE=$(aws elbv2 describe-target-health           --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupSecurityhq530DEDAA",
+                  },
+                  "           --region ",
+                  {
+                    "Ref": "AWS::Region",
+                  },
+                  "           --targets Id=$INSTANCE_ID,Port=9000           --query "TargetHealthDescriptions[0].TargetHealth.State")
+      done
 
-    dpkg -i /tmp/installer.deb",
+      echo "Instance is healthy in target group."
+
+# GuEc2AppExperimental UserData End",
                 ],
               ],
             },

cdk/lib/security-hq.test.ts

@@ -8,6 +8,7 @@ describe("HQ stack", () => {
     const stack = new SecurityHQ(app, "security-hq", {
       stack: "security",
       stage: "PROD",
+      buildIdentifier: "TEST"
     });
     expect(Template.fromStack(stack).toJSON()).toMatchSnapshot();
   });

cdk/lib/security-hq.ts

@@ -1,4 +1,3 @@
-import { GuEc2App } from "@guardian/cdk";
 import { AccessScope } from "@guardian/cdk/lib/constants";
 import { GuAlarm } from "@guardian/cdk/lib/constructs/cloudwatch";
 import type { GuStackProps } from "@guardian/cdk/lib/constructs/core";
@@ -20,6 +19,7 @@ import {
   GuPutCloudwatchMetricsPolicy,
 } from "@guardian/cdk/lib/constructs/iam";
 import { GuAnghammaradSenderPolicy } from "@guardian/cdk/lib/constructs/iam/policies/anghammarad";
+import { GuEc2AppExperimental } from "@guardian/cdk/lib/experimental/patterns/ec2-app";
 import { Duration, RemovalPolicy, SecretValue } from "aws-cdk-lib";
 import type { App } from "aws-cdk-lib";
 import {
@@ -41,14 +41,24 @@ import {
   StringParameter,
 } from "aws-cdk-lib/aws-ssm";
 
+interface SecurityHQProps extends GuStackProps {
+  /**
+   * Which application build to run.
+   * This will typically match the build number provided by CI.
+   */
+  buildIdentifier: string;
+}
+
 export class SecurityHQ extends GuStack {
   private static app: AppIdentity = {
     app: "security-hq",
   };
 
-  constructor(scope: App, id: string, props: GuStackProps) {
+  constructor(scope: App, id: string, props: SecurityHQProps) {
     super(scope, id, props);
 
+    const { buildIdentifier } = props;
+
     const table = new GuDynamoTable(this, "DynamoTable", {
       tableName: `security-hq-iam`,
       removalPolicy: RemovalPolicy.RETAIN,
@@ -92,11 +102,11 @@ export class SecurityHQ extends GuStack {
 
     aws --region eu-west-1 s3 cp s3://${distBucket.valueAsString}/security/${this.stage}/security-hq/security-hq.conf /etc/gu
     aws --region eu-west-1 s3 cp s3://${distBucket.valueAsString}/security/${this.stage}/security-hq/security-hq-service-account-cert.json /etc/gu
-    aws --region eu-west-1 s3 cp s3://${distBucket.valueAsString}/security/${this.stage}/security-hq/security-hq.deb /tmp/installer.deb
+    aws --region eu-west-1 s3 cp s3://${distBucket.valueAsString}/security/${this.stage}/security-hq/security-hq-${buildIdentifier}.deb /tmp/installer.deb
 
     dpkg -i /tmp/installer.deb`);
 
-    const ec2App = new GuEc2App(this, {
+    const ec2App = new GuEc2AppExperimental(this, {
       applicationLogging: {
         enabled: true
       },