diff --git a/.changeset/empty-shirts-film.md b/.changeset/empty-shirts-film.md new file mode 100644 index 000000000..a330e1ad0 --- /dev/null +++ b/.changeset/empty-shirts-film.md @@ -0,0 +1,5 @@ +--- +'skuba': minor +--- + +**template/lambda-sqs-worker-cdk:** Add new template diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e1d470e26..5d4191bfa 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -53,6 +53,7 @@ jobs: - greeter - koa-rest-api - lambda-sqs-worker + - lambda-sqs-worker-cdk - oss-npm-package - private-npm-package steps: diff --git a/README.md b/README.md index e56c2cc69..4ef24e94c 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,16 @@ This initialises a new directory and Git repository. a message queue is employed between the source topic and the Lambda function, and unprocessed events are sent to a dead-letter queue for manual triage. +- `lambda-sqs-worker-cdk` + + An asynchronous [worker] built on [AWS Lambda] and deployed with [AWS CDK]. + + ```text + SNS -> SQS (with a dead-letter queue) -> Lambda + ``` + + Comes with configuration validation and infrastructure snapshot testing. + - `oss-npm-package` A public npm package published via [semantic-release] pipeline. @@ -197,6 +207,7 @@ This initialises a new directory and Git repository. [serverless]: https://serverless.com/ [worker]: https://tech-strategy.ssod.skinfra.xyz/docs/v1/components.html#worker [express]: https://expressjs.com/ +[aws cdk]: https://tech-strategy.ssod.skinfra.xyz/docs/v1/technology.html#cdk This script is interactive by default. For unattended execution, pipe in JSON: diff --git a/config/tsconfig.json b/config/tsconfig.json index 1b5fa092d..ab373abba 100644 --- a/config/tsconfig.json +++ b/config/tsconfig.json @@ -1,7 +1,8 @@ { "compilerOptions": { "incremental": true, - "moduleResolution": "node" + "moduleResolution": "node", + "resolveJsonModule": true }, "extends": "tsconfig-seek" } diff --git a/src/cli/configure/analysis/__snapshots__/project.test.ts.snap b/src/cli/configure/analysis/__snapshots__/project.test.ts.snap index 3e9363fbd..8459f33e4 100644 --- a/src/cli/configure/analysis/__snapshots__/project.test.ts.snap +++ b/src/cli/configure/analysis/__snapshots__/project.test.ts.snap @@ -55,9 +55,11 @@ node_modules*/ }, ".gitignore": Object { "data": "# managed by skuba +.cdk.staging/ .idea/ .serverless/ .vscode/ +cdk.out/ node_modules*/ /coverage*/ diff --git a/src/cli/init/prompts.ts b/src/cli/init/prompts.ts index a24e29060..402bfe8b9 100644 --- a/src/cli/init/prompts.ts +++ b/src/cli/init/prompts.ts @@ -73,6 +73,7 @@ export const TEMPLATE_PROMPT = new Select({ 'greeter', 'koa-rest-api', 'lambda-sqs-worker', + 'lambda-sqs-worker-cdk', 'oss-npm-package', 'private-npm-package', 'github →', diff --git a/template/base/_.gitignore b/template/base/_.gitignore index 67b6f5832..7b07d4ca9 100644 --- a/template/base/_.gitignore +++ b/template/base/_.gitignore @@ -1,7 +1,9 @@ # managed by skuba +.cdk.staging/ .idea/ .serverless/ .vscode/ +cdk.out/ node_modules*/ /coverage*/ diff --git a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml new file mode 100644 index 000000000..08087633f --- /dev/null +++ b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml @@ -0,0 +1,61 @@ +dev-agent: &dev-agent + agents: + queue: <%- devBuildkiteQueueName %> + +prod-agent: &prod-agent + agents: + queue: <%- prodBuildkiteQueueName %> + +plugins: &plugins #alias for shared plugins + seek-oss/aws-sm#v2.3.1: + env: + NPM_TOKEN: 'arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token' + docker#v3.8.0: + volumes: + - /workdir/node_modules + - /workdir/lib + environment: + - NPM_TOKEN + seek-oss/docker-ecr-cache#v1.9.0: + build-args: + - NPM_TOKEN + cache-on: + - package.json + - yarn.lock + +steps: + - label: ':yarn: :eslint: Lint and :jest: unit test' + <<: *dev-agent + plugins: + <<: *plugins + key: test + command: + - echo "--- Running yarn lint" + - yarn lint + - echo "--- Running yarn test :jest:" + - yarn test + + - label: 'CDK Deploy Staging :shipit:' + <<: *dev-agent + plugins: + <<: *plugins + depends_on: + - test + command: + - echo "--- Running CDK deploy to staging" + - yarn deploy:dev + concurrency: 1 + concurrency_group: '<%- repoName %>/deploy/dev' + + - label: 'CDK Deploy Production :shipit:' + branches: 'master' + <<: *prod-agent + plugins: + <<: *plugins + depends_on: + - test + command: + - echo "--- Running CDK deploy to production" + - yarn deploy:prod + concurrency: 1 + concurrency_group: '<%- repoName %>/deploy/prod' diff --git a/template/lambda-sqs-worker-cdk/.me b/template/lambda-sqs-worker-cdk/.me new file mode 100644 index 000000000..cb4347112 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/.me @@ -0,0 +1,22 @@ +--- +# https://codex.ssod.skinfra.xyz/docs + +components: + '<%- repoName %>': + # TODO: supply system catalog information + # dependencies: + # - type: api + # key: SEEK-Jobs/??? + # - type: datastore + # arn: arn:aws:dynamodb:us-east-1:123456789012:table/??? + # - type: datastore + # key: infrastructure/??? + deploy_target: Worker + # is_production_system: true + primary_technologies: + - AWS CDK + - AWS Lambda + - Buildkite + - skuba + - TypeScript + # scope: APAC diff --git a/template/lambda-sqs-worker-cdk/.nvmrc b/template/lambda-sqs-worker-cdk/.nvmrc new file mode 100644 index 000000000..8351c1939 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/.nvmrc @@ -0,0 +1 @@ +14 diff --git a/template/lambda-sqs-worker-cdk/Dockerfile b/template/lambda-sqs-worker-cdk/Dockerfile new file mode 100644 index 000000000..1ef531c54 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/Dockerfile @@ -0,0 +1,30 @@ +# Docker image history includes ARG values, so never target this stage directly +FROM node:14-alpine AS unsafe-dev-deps + +WORKDIR /workdir + +COPY package.json yarn.lock ./ + +ARG NPM_READ_TOKEN + +RUN \ + echo '//registry.npmjs.org/:_authToken=${NPM_READ_TOKEN}' > .npmrc && \ + yarn install --frozen-lockfile --ignore-optional --non-interactive && \ + yarn package && \ + rm .npmrc + +### + +FROM node:14-alpine AS dev-deps + +WORKDIR /workdir + +COPY --from=unsafe-dev-deps /workdir . + +### + +FROM dev-deps AS build + +COPY . . + +RUN yarn build diff --git a/template/lambda-sqs-worker-cdk/cdk.json b/template/lambda-sqs-worker-cdk/cdk.json new file mode 100644 index 000000000..0a2c70d9a --- /dev/null +++ b/template/lambda-sqs-worker-cdk/cdk.json @@ -0,0 +1,25 @@ +{ + "app": "npx ts-node infra/index.ts", + "context": { + "@aws-cdk/core:enableStackNameDuplicates": "true", + "global": { + "appName": "<%- serviceName %>" + }, + "dev": { + "workerLambda": { + "reservedConcurrency": 1, + "environment": { + "SOMETHING": "dev" + } + } + }, + "prod": { + "workerLambda": { + "reservedConcurrency": 2, + "environment": { + "SOMETHING": "prod" + } + } + } + } +} diff --git a/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap b/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap new file mode 100644 index 000000000..1c533e86a --- /dev/null +++ b/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap @@ -0,0 +1,821 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`returns expected cloud formation stack 1`] = ` +Object { + "Parameters": Object { + "AssetParameters...": Object { + "Description": "Artifact hash for asset...", + "Type": "String", + }, + }, + "Resources": Object { + "kmskey49FBC3B3": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "Description": "serviceName", + "EnableKeyRotation": true, + "KeyPolicy": Object { + "Statement": Array [ + Object { + "Action": Array [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + "kms:GenerateDataKey", + "kms:TagResource", + "kms:UntagResource", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": Array [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": Array [ + "kms:Encrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::GetAtt": Array [ + "workerServiceRole2130CC7F", + "Arn", + ], + }, + }, + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + }, + "Type": "AWS::KMS::Key", + "UpdateReplacePolicy": "Retain", + }, + "kmskeyAlias39245779": Object { + "Properties": Object { + "AliasName": "alias/seek/self/serviceName", + "TargetKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + }, + "Type": "AWS::KMS::Alias", + }, + "topic69831491": Object { + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "TopicName": "serviceName", + }, + "Type": "AWS::SNS::Topic", + }, + "worker28EA3E30": Object { + "DependsOn": Array [ + "workerServiceRoleDefaultPolicyBA498553", + "workerServiceRole2130CC7F", + ], + "Properties": Object { + "Code": Object { + "S3Bucket": Object { + "Ref": "AssetParameters...", + }, + "S3Key": Object { + "Fn::Join": Array [ + "", + Array [ + Object { + "Fn::Select": Array [ + 0, + Object { + "Fn::Split": Array [ + "||", + Object { + "Ref": "AssetParameters...", + }, + ], + }, + ], + }, + Object { + "Fn::Select": Array [ + 1, + Object { + "Fn::Split": Array [ + "||", + Object { + "Ref": "AssetParameters...", + }, + ], + }, + ], + }, + ], + ], + }, + }, + "Environment": Object { + "Variables": Object { + "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1", + "SOMETHING": "dev", + }, + }, + "FunctionName": "serviceName", + "Handler": "app.handler", + "KmsKeyArn": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "Role": Object { + "Fn::GetAtt": Array [ + "workerServiceRole2130CC7F", + "Arn", + ], + }, + "Runtime": "nodejs14.x", + }, + "Type": "AWS::Lambda::Function", + }, + "workerServiceRole2130CC7F": Object { + "Properties": Object { + "AssumeRolePolicyDocument": Object { + "Statement": Array [ + Object { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": Object { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": Array [ + Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "workerServiceRoleDefaultPolicyBA498553": Object { + "Properties": Object { + "PolicyDocument": Object { + "Statement": Array [ + Object { + "Action": Array [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + }, + Object { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "workerServiceRoleDefaultPolicyBA498553", + "Roles": Array [ + Object { + "Ref": "workerServiceRole2130CC7F", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "workerSqsEventSourceappStackworkerqueue8281B9F47B9F582B": Object { + "Properties": Object { + "EventSourceArn": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + "FunctionName": Object { + "Ref": "worker28EA3E30", + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "workerqueueA05CE5C6": Object { + "DeletionPolicy": "Delete", + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "QueueName": "serviceName", + "RedrivePolicy": Object { + "deadLetterTargetArn": Object { + "Fn::GetAtt": Array [ + "workerqueuedlq42262778", + "Arn", + ], + }, + "maxReceiveCount": 3, + }, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "workerqueuePolicy97054CB4": Object { + "Properties": Object { + "PolicyDocument": Object { + "Statement": Array [ + Object { + "Action": "sqs:SendMessage", + "Condition": Object { + "ArnEquals": Object { + "aws:SourceArn": Object { + "Ref": "topic69831491", + }, + }, + }, + "Effect": "Allow", + "Principal": Object { + "Service": "sns.amazonaws.com", + }, + "Resource": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "Queues": Array [ + Object { + "Ref": "workerqueueA05CE5C6", + }, + ], + }, + "Type": "AWS::SQS::QueuePolicy", + }, + "workerqueueappStacktopic0CA45134AFB31FF4": Object { + "Properties": Object { + "Endpoint": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + "Protocol": "sqs", + "TopicArn": Object { + "Ref": "topic69831491", + }, + }, + "Type": "AWS::SNS::Subscription", + }, + "workerqueuedlq42262778": Object { + "DeletionPolicy": "Delete", + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "QueueName": "serviceName-dlq", + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + }, +} +`; + +exports[`returns expected cloud formation stack 2`] = ` +Object { + "Parameters": Object { + "AssetParameters...": Object { + "Description": "Artifact hash for asset...", + "Type": "String", + }, + }, + "Resources": Object { + "kmskey49FBC3B3": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "Description": "serviceName", + "EnableKeyRotation": true, + "KeyPolicy": Object { + "Statement": Array [ + Object { + "Action": Array [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + "kms:GenerateDataKey", + "kms:TagResource", + "kms:UntagResource", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": Array [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": Array [ + "kms:Encrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + ], + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::", + Object { + "Ref": "AWS::AccountId", + }, + ":root", + ], + ], + }, + }, + "Resource": "*", + }, + Object { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Principal": Object { + "AWS": Object { + "Fn::GetAtt": Array [ + "workerServiceRole2130CC7F", + "Arn", + ], + }, + }, + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + }, + "Type": "AWS::KMS::Key", + "UpdateReplacePolicy": "Retain", + }, + "kmskeyAlias39245779": Object { + "Properties": Object { + "AliasName": "alias/seek/self/serviceName", + "TargetKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + }, + "Type": "AWS::KMS::Alias", + }, + "topic69831491": Object { + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "TopicName": "serviceName", + }, + "Type": "AWS::SNS::Topic", + }, + "worker28EA3E30": Object { + "DependsOn": Array [ + "workerServiceRoleDefaultPolicyBA498553", + "workerServiceRole2130CC7F", + ], + "Properties": Object { + "Code": Object { + "S3Bucket": Object { + "Ref": "AssetParameters...", + }, + "S3Key": Object { + "Fn::Join": Array [ + "", + Array [ + Object { + "Fn::Select": Array [ + 0, + Object { + "Fn::Split": Array [ + "||", + Object { + "Ref": "AssetParameters...", + }, + ], + }, + ], + }, + Object { + "Fn::Select": Array [ + 1, + Object { + "Fn::Split": Array [ + "||", + Object { + "Ref": "AssetParameters...", + }, + ], + }, + ], + }, + ], + ], + }, + }, + "Environment": Object { + "Variables": Object { + "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1", + "SOMETHING": "prod", + }, + }, + "FunctionName": "serviceName", + "Handler": "app.handler", + "KmsKeyArn": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "Role": Object { + "Fn::GetAtt": Array [ + "workerServiceRole2130CC7F", + "Arn", + ], + }, + "Runtime": "nodejs14.x", + }, + "Type": "AWS::Lambda::Function", + }, + "workerServiceRole2130CC7F": Object { + "Properties": Object { + "AssumeRolePolicyDocument": Object { + "Statement": Array [ + Object { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": Object { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": Array [ + Object { + "Fn::Join": Array [ + "", + Array [ + "arn:", + Object { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "workerServiceRoleDefaultPolicyBA498553": Object { + "Properties": Object { + "PolicyDocument": Object { + "Statement": Array [ + Object { + "Action": Array [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + }, + Object { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "workerServiceRoleDefaultPolicyBA498553", + "Roles": Array [ + Object { + "Ref": "workerServiceRole2130CC7F", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "workerSqsEventSourceappStackworkerqueue8281B9F47B9F582B": Object { + "Properties": Object { + "EventSourceArn": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + "FunctionName": Object { + "Ref": "worker28EA3E30", + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "workerqueueA05CE5C6": Object { + "DeletionPolicy": "Delete", + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "QueueName": "serviceName", + "RedrivePolicy": Object { + "deadLetterTargetArn": Object { + "Fn::GetAtt": Array [ + "workerqueuedlq42262778", + "Arn", + ], + }, + "maxReceiveCount": 3, + }, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "workerqueuePolicy97054CB4": Object { + "Properties": Object { + "PolicyDocument": Object { + "Statement": Array [ + Object { + "Action": "sqs:SendMessage", + "Condition": Object { + "ArnEquals": Object { + "aws:SourceArn": Object { + "Ref": "topic69831491", + }, + }, + }, + "Effect": "Allow", + "Principal": Object { + "Service": "sns.amazonaws.com", + }, + "Resource": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "Queues": Array [ + Object { + "Ref": "workerqueueA05CE5C6", + }, + ], + }, + "Type": "AWS::SQS::QueuePolicy", + }, + "workerqueueappStacktopic0CA45134AFB31FF4": Object { + "Properties": Object { + "Endpoint": Object { + "Fn::GetAtt": Array [ + "workerqueueA05CE5C6", + "Arn", + ], + }, + "Protocol": "sqs", + "TopicArn": Object { + "Ref": "topic69831491", + }, + }, + "Type": "AWS::SNS::Subscription", + }, + "workerqueuedlq42262778": Object { + "DeletionPolicy": "Delete", + "Properties": Object { + "KmsMasterKeyId": Object { + "Fn::GetAtt": Array [ + "kmskey49FBC3B3", + "Arn", + ], + }, + "QueueName": "serviceName-dlq", + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + }, +} +`; diff --git a/template/lambda-sqs-worker-cdk/infra/appStack.test.ts b/template/lambda-sqs-worker-cdk/infra/appStack.test.ts new file mode 100644 index 000000000..2f554e5a6 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/infra/appStack.test.ts @@ -0,0 +1,29 @@ +import { SynthUtils } from '@aws-cdk/assert'; +import { App } from '@aws-cdk/core'; + +import cdkJson from '../cdk.json'; + +import { AppStack } from './appStack'; + +const contexts = [ + { + stage: 'dev', + ...cdkJson.context, + }, + + { + stage: 'prod', + ...cdkJson.context, + }, +]; + +it.each(contexts)('returns expected cloud formation stack', (context) => { + const app = new App({ context }); + + const stack = new AppStack(app, 'appStack'); + + const json = JSON.stringify(SynthUtils.toCloudFormation(stack)) + .replace(/AssetParameters[a-zA-Z0-9]+/gm, 'AssetParameters...') + .replace(/"Artifact hash for asset .+"/gm, '"Artifact hash for asset..."'); + expect(JSON.parse(json)).toMatchSnapshot(); +}); diff --git a/template/lambda-sqs-worker-cdk/infra/appStack.ts b/template/lambda-sqs-worker-cdk/infra/appStack.ts new file mode 100644 index 000000000..6067b2f09 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/infra/appStack.ts @@ -0,0 +1,65 @@ +import { AccountPrincipal } from '@aws-cdk/aws-iam'; +import { Key } from '@aws-cdk/aws-kms'; +import { AssetCode, Function, Runtime } from '@aws-cdk/aws-lambda'; +import { SqsEventSource } from '@aws-cdk/aws-lambda-event-sources'; +import { Topic } from '@aws-cdk/aws-sns'; +import { SqsSubscription } from '@aws-cdk/aws-sns-subscriptions'; +import { Queue } from '@aws-cdk/aws-sqs'; +import { Construct, Stack, StackProps } from '@aws-cdk/core'; + +import { envContext, stageContext } from '../shared/context-types'; + +export class AppStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + const stage = stageContext.check(this.node.tryGetContext('stage')); + const context = envContext.check(this.node.tryGetContext(stage)); + + const accountPrincipal = new AccountPrincipal(this.account); + + const kmsKey = new Key(this, 'kms-key', { + description: '<%- serviceName %>', + enableKeyRotation: true, + admins: [accountPrincipal], + alias: 'seek/self/<%- serviceName %>', + }); + + kmsKey.grantEncrypt(accountPrincipal); + + const topic = new Topic(this, 'topic', { + topicName: '<%- serviceName %>', + masterKey: kmsKey, + }); + + const deadLetterQueue = new Queue(this, 'worker-queue-dlq', { + queueName: '<%- serviceName %>-dlq', + encryptionMasterKey: kmsKey, + }); + + const queue = new Queue(this, 'worker-queue', { + queueName: '<%- serviceName %>', + deadLetterQueue: { + maxReceiveCount: 3, + queue: deadLetterQueue, + }, + encryptionMasterKey: kmsKey, + }); + + const worker = new Function(this, 'worker', { + code: new AssetCode('./lib'), + runtime: Runtime.NODEJS_14_X, + handler: 'app.handler', + functionName: '<%- serviceName %>', + environmentEncryption: kmsKey, + environment: { + AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', + ...context.workerLambda.environment, + }, + }); + + worker.addEventSource(new SqsEventSource(queue)); + + topic.addSubscription(new SqsSubscription(queue)); + } +} diff --git a/template/lambda-sqs-worker-cdk/infra/index.ts b/template/lambda-sqs-worker-cdk/infra/index.ts new file mode 100644 index 000000000..7d9cf7d7b --- /dev/null +++ b/template/lambda-sqs-worker-cdk/infra/index.ts @@ -0,0 +1,14 @@ +/* eslint-disable no-new */ +import { App } from '@aws-cdk/core'; + +import { globalContext } from '../shared/context-types'; + +import { AppStack } from './appStack'; + +const app = new App(); + +const context = globalContext.check(app.node.tryGetContext('global')); + +new AppStack(app, 'appStack', { + stackName: context.appName, +}); diff --git a/template/lambda-sqs-worker-cdk/package.json b/template/lambda-sqs-worker-cdk/package.json new file mode 100644 index 000000000..a066037cf --- /dev/null +++ b/template/lambda-sqs-worker-cdk/package.json @@ -0,0 +1,32 @@ +{ + "dependencies": { + "@seek/logger": "^4.4.7", + "runtypes": "^6.0.0" + }, + "devDependencies": { + "@aws-cdk/assert": "^1.95.1", + "@aws-cdk/aws-iam": "^1.95.1", + "@aws-cdk/aws-kms": "^1.95.1", + "@aws-cdk/aws-lambda": "^1.95.1", + "@aws-cdk/aws-lambda-event-sources": "^1.95.1", + "@aws-cdk/aws-sns": "^1.95.1", + "@aws-cdk/aws-sns-subscriptions": "^1.95.1", + "@aws-cdk/aws-sqs": "^1.95.1", + "@aws-cdk/core": "^1.95.1", + "@types/aws-lambda": "^8.10.73", + "@types/node": "^14.14.37", + "aws-cdk": "^1.18.0", + "skuba": "*" + }, + "license": "UNLICENSED", + "private": true, + "scripts": { + "build": "skuba build", + "format": "skuba format", + "lint": "skuba lint", + "test": "skuba test", + "package": "yarn --prod --modules-folder ./lib/node_modules", + "deploy:dev": "cdk deploy appStack --require-approval never --context stage=dev", + "deploy:prod": "cdk deploy appStack --require-approval never --context stage=prod" + } +} diff --git a/template/lambda-sqs-worker-cdk/shared/context-types.ts b/template/lambda-sqs-worker-cdk/shared/context-types.ts new file mode 100644 index 000000000..10ac89471 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/shared/context-types.ts @@ -0,0 +1,30 @@ +/* eslint-disable new-cap */ +import * as t from 'runtypes'; + +export const stageContext = t.Union(t.Literal('dev'), t.Literal('prod')); +export type StageContext = t.Static; + +export const envContext = t + .Record({ + workerLambda: t + .Record({ + reservedConcurrency: t.Number, + environment: t + .Record({ + SOMETHING: t.String, + }) + .asReadonly(), + }) + .asReadonly(), + }) + .asReadonly(); + +export type EnvContext = t.Static; + +export const globalContext = t + .Record({ + appName: t.String, + }) + .asReadonly(); + +export type GlobalContext = t.Static; diff --git a/template/lambda-sqs-worker-cdk/skuba.template.js b/template/lambda-sqs-worker-cdk/skuba.template.js new file mode 100644 index 000000000..55d8cd7fa --- /dev/null +++ b/template/lambda-sqs-worker-cdk/skuba.template.js @@ -0,0 +1,27 @@ +/** + * Run `skuba configure` to finish templating and remove this file. + */ + +module.exports = { + entryPoint: 'src/app.ts#handler', + fields: [ + { + name: 'serviceName', + message: 'Service slug', + initial: 'my-project', + }, + { + name: 'devBuildkiteQueueName', + message: 'Dev Buildkite queue', + initial: 'my-team-aws-account-dev:cicd', + validate: (value) => /^.+:.+$/.test(value), + }, + { + name: 'prodBuildkiteQueueName', + message: 'Prod Buildkite queue', + initial: 'my-team-aws-account-prod:cicd', + validate: (value) => /^.+:.+$/.test(value), + }, + ], + type: 'application', +}; diff --git a/template/lambda-sqs-worker-cdk/src/app.ts b/template/lambda-sqs-worker-cdk/src/app.ts new file mode 100644 index 000000000..de0e6ebd1 --- /dev/null +++ b/template/lambda-sqs-worker-cdk/src/app.ts @@ -0,0 +1,10 @@ +import createLogger from '@seek/logger'; +import { SQSEvent, SQSHandler } from 'aws-lambda'; + +const logger = createLogger({ + name: '<%- serviceName %>', +}); + +export const handler: SQSHandler = (_: SQSEvent) => { + logger.info('Hello World!'); +};