OpenSearch Service CDK setup (#14: lewijacn/opensearch-migrations#14)

MIGRATIONS-876: Provide an initial IaC CDK Managed OpenSearch Service solution for users

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>

.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

bin/app.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import {App} from 'aws-cdk-lib';
+import {StackComposer} from "../lib/stack-composer";
+
+const app = new App();
+const stage = process.env.CDK_DEPLOYMENT_STAGE
+const account = process.env.CDK_DEFAULT_ACCOUNT
+const region = process.env.CDK_DEFAULT_REGION
+new StackComposer(app, {
+    env: { account: account, region: region },
+    stackName: `OSServiceDomainCDKStack-${stage}-${region}`,
+    description: "This stack contains resources to create/manage an OpenSearch Service domain"
+});

cdk.json

@@ -0,0 +1,44 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false
+  }
+}

default-values.json

@@ -0,0 +1,4 @@
+{
+  "engineVersion": "OS_1.3",
+  "domainName": "os-service-domain"
+}

jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

lib/opensearch-service-domain-cdk-stack.ts

@@ -0,0 +1,100 @@
+import { Construct } from 'constructs';
+import {EbsDeviceVolumeType, IVpc, Vpc} from "aws-cdk-lib/aws-ec2";
+import {Domain, EngineVersion, TLSSecurityPolicy} from "aws-cdk-lib/aws-opensearchservice";
+import {RemovalPolicy, SecretValue, Stack, StackProps} from "aws-cdk-lib";
+import {IKey, Key} from "aws-cdk-lib/aws-kms";
+import {PolicyStatement} from "aws-cdk-lib/aws-iam";
+import {ILogGroup, LogGroup} from "aws-cdk-lib/aws-logs";
+import {Secret} from "aws-cdk-lib/aws-secretsmanager";
+
+
+export interface opensearchServiceDomainCdkProps extends StackProps{
+  readonly version: EngineVersion,
+  readonly domainName: string,
+  readonly dataNodeInstanceType?: string,
+  readonly dataNodes?: number,
+  readonly dedicatedManagerNodeType?: string,
+  readonly dedicatedManagerNodeCount?: number,
+  readonly warmInstanceType?: string,
+  readonly warmNodes?: number
+  readonly accessPolicies?: PolicyStatement[],
+  readonly useUnsignedBasicAuth?: boolean,
+  readonly fineGrainedManagerUserARN?: string,
+  readonly fineGrainedManagerUserName?: string,
+  readonly fineGrainedManagerUserSecretManagerKeyARN?: string,
+  readonly enforceHTTPS?: boolean,
+  readonly tlsSecurityPolicy?: TLSSecurityPolicy,
+  readonly ebsEnabled?: boolean,
+  readonly ebsIops?: number,
+  readonly ebsVolumeSize?: number,
+  readonly ebsVolumeType?: EbsDeviceVolumeType,
+  readonly encryptionAtRestEnabled?: boolean,
+  readonly encryptionAtRestKmsKeyARN?: string,
+  readonly appLogEnabled?: boolean,
+  readonly appLogGroup?: string,
+  readonly nodeToNodeEncryptionEnabled?: boolean,
+  readonly vpcId?: string,
+  readonly domainRemovalPolicy?: RemovalPolicy
+}
+
+
+export class OpensearchServiceDomainCdkStack extends Stack {
+  constructor(scope: Construct, id: string, props: opensearchServiceDomainCdkProps) {
+    super(scope, id, props);
+
+    // The code that defines your stack goes here
+
+    // Retrieve existing account resources if defined
+    const earKmsKey: IKey|undefined = props.encryptionAtRestKmsKeyARN && props.encryptionAtRestEnabled ?
+        Key.fromKeyArn(this, "earKey", props.encryptionAtRestKmsKeyARN) : undefined
+
+    const managerUserSecret: SecretValue|undefined = props.fineGrainedManagerUserSecretManagerKeyARN ?
+        Secret.fromSecretCompleteArn(this, "managerSecret", props.fineGrainedManagerUserSecretManagerKeyARN).secretValue : undefined
+
+    const appLG: ILogGroup|undefined = props.appLogGroup && props.appLogEnabled ?
+        LogGroup.fromLogGroupArn(this, "appLogGroup", props.appLogGroup) : undefined
+
+    const vpc: IVpc|undefined = props.vpcId ?
+        Vpc.fromLookup(this, "domainVPC", {vpcId: props.vpcId}) : undefined
+
+
+    const domain = new Domain(this, 'Domain', {
+      version: props.version,
+      domainName: props.domainName,
+      accessPolicies: props.accessPolicies,
+      useUnsignedBasicAuth: props.useUnsignedBasicAuth,
+      capacity: {
+        dataNodeInstanceType: props.dataNodeInstanceType,
+        dataNodes: props.dataNodes,
+        masterNodeInstanceType: props.dedicatedManagerNodeType,
+        masterNodes: props.dedicatedManagerNodeCount,
+        warmInstanceType: props.warmInstanceType,
+        warmNodes: props.warmNodes
+      },
+      fineGrainedAccessControl: {
+        masterUserArn: props.fineGrainedManagerUserARN,
+        masterUserName: props.fineGrainedManagerUserName,
+        masterUserPassword: managerUserSecret
+      },
+      nodeToNodeEncryption: props.nodeToNodeEncryptionEnabled,
+      encryptionAtRest: {
+        enabled: props.encryptionAtRestEnabled,
+        kmsKey: earKmsKey
+      },
+      enforceHttps: props.enforceHTTPS,
+      tlsSecurityPolicy: props.tlsSecurityPolicy,
+      ebs: {
+        enabled: props.ebsEnabled,
+        iops: props.ebsIops,
+        volumeSize: props.ebsVolumeSize,
+        volumeType: props.ebsVolumeType
+      },
+      logging: {
+        appLogEnabled: props.appLogEnabled,
+        appLogGroup: appLG
+      },
+      vpc: vpc,
+      removalPolicy: props.domainRemovalPolicy
+    });
+  }
+}

lib/stack-composer.ts

@@ -0,0 +1,147 @@
+import {Construct} from "constructs";
+import {RemovalPolicy, Stack, StackProps} from "aws-cdk-lib";
+import {OpensearchServiceDomainCdkStack} from "./opensearch-service-domain-cdk-stack";
+import {EngineVersion, TLSSecurityPolicy} from "aws-cdk-lib/aws-opensearchservice";
+import {EbsDeviceVolumeType} from "aws-cdk-lib/aws-ec2";
+import {PolicyStatement} from "aws-cdk-lib/aws-iam";
+import * as defaultValuesJson from "../default-values.json"
+
+export class StackComposer {
+    public stacks: Stack[] = [];
+
+    constructor(scope: Construct, props: StackProps) {
+
+        let version: EngineVersion
+
+        const defaultValues: { [x: string]: (string); } = defaultValuesJson
+        const domainName = getContextForType('domainName', 'string')
+        const dataNodeType = getContextForType('dataNodeType', 'string')
+        const dataNodeCount = getContextForType('dataNodeCount', 'number')
+        const dedicatedManagerNodeType = getContextForType('dedicatedManagerNodeType', 'string')
+        const dedicatedManagerNodeCount = getContextForType('dedicatedManagerNodeCount', 'number')
+        const warmNodeType = getContextForType('warmNodeType', 'string')
+        const warmNodeCount = getContextForType('warmNodeCount', 'number')
+        const useUnsignedBasicAuth = getContextForType('useUnsignedBasicAuth', 'boolean')
+        const fineGrainedManagerUserARN = getContextForType('fineGrainedManagerUserARN', 'string')
+        const fineGrainedManagerUserName = getContextForType('fineGrainedManagerUserName', 'string')
+        const fineGrainedManagerUserSecretManagerKeyARN = getContextForType('fineGrainedManagerUserSecretManagerKeyARN', 'string')
+        const enforceHTTPS = getContextForType('enforceHTTPS', 'boolean')
+        const ebsEnabled = getContextForType('ebsEnabled', 'boolean')
+        const ebsIops = getContextForType('ebsIops', 'number')
+        const ebsVolumeSize = getContextForType('ebsVolumeSize', 'number')
+        const encryptionAtRestEnabled = getContextForType('encryptionAtRestEnabled', 'boolean')
+        const encryptionAtRestKmsKeyARN = getContextForType("encryptionAtRestKmsKeyARN", 'string')
+        const loggingAppLogEnabled = getContextForType('loggingAppLogEnabled', 'boolean')
+        const loggingAppLogGroupARN = getContextForType('loggingAppLogGroupARN', 'string')
+        const noneToNodeEncryptionEnabled = getContextForType('nodeToNodeEncryptionEnabled', 'boolean')
+        const vpcId = getContextForType('vpcId', 'string')
+
+        if (!domainName) {
+            throw new Error("Domain name is not present and is a required field")
+        }
+
+        const engineVersion = getContextForType('engineVersion', 'string')
+        if (engineVersion && engineVersion.startsWith("OS_")) {
+            // Will accept a period delimited version string (i.e. 1.3) and return a proper EngineVersion
+            version = EngineVersion.openSearch(engineVersion.substring(3))
+        }
+        else if (engineVersion && engineVersion.startsWith("ES_")) {
+            version = EngineVersion.elasticsearch(engineVersion.substring(3))
+        }
+        else {
+            throw new Error("Engine version is not present or does not match the expected format, i.e. OS_1.3 or ES_7.9")
+        }
+
+        const accessPolicyJson = getContextForType('accessPolicies', 'object')
+        const accessPolicies = accessPolicyJson ? parseAccessPolicies(accessPolicyJson) : undefined
+
+        const tlsSecurityPolicyName = getContextForType('tlsSecurityPolicy', 'string')
+        const tlsSecurityPolicy: TLSSecurityPolicy|undefined = tlsSecurityPolicyName ? TLSSecurityPolicy[tlsSecurityPolicyName as keyof typeof TLSSecurityPolicy] : undefined
+        if (tlsSecurityPolicyName && !tlsSecurityPolicy) {
+            throw new Error("Provided tlsSecurityPolicy does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_opensearchservice.TLSSecurityPolicy.html")
+        }
+
+        const ebsVolumeTypeName = getContextForType('ebsVolumeType', 'string')
+        const ebsVolumeType: EbsDeviceVolumeType|undefined = ebsVolumeTypeName ? EbsDeviceVolumeType[ebsVolumeTypeName as keyof typeof EbsDeviceVolumeType] : undefined
+        if (ebsVolumeTypeName && !ebsVolumeType) {
+            throw new Error("Provided ebsVolumeType does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.EbsDeviceVolumeType.html")
+        }
+
+        const domainRemovalPolicyName = getContextForType('domainRemovalPolicy', 'string')
+        const domainRemovalPolicy = domainRemovalPolicyName ? RemovalPolicy[domainRemovalPolicyName as keyof typeof RemovalPolicy] : undefined
+        if (domainRemovalPolicyName && !domainRemovalPolicy) {
+            throw new Error("Provided domainRemovalPolicy does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html")
+        }
+
+        const opensearchStack = new OpensearchServiceDomainCdkStack(scope, 'opensearchDomainStack', {
+            version: version,
+            domainName: domainName,
+            dataNodeInstanceType: dataNodeType,
+            dataNodes: dataNodeCount,
+            dedicatedManagerNodeType: dedicatedManagerNodeType,
+            dedicatedManagerNodeCount: dedicatedManagerNodeCount,
+            warmInstanceType: warmNodeType,
+            warmNodes: warmNodeCount,
+            accessPolicies: accessPolicies,
+            useUnsignedBasicAuth: useUnsignedBasicAuth,
+            fineGrainedManagerUserARN: fineGrainedManagerUserARN,
+            fineGrainedManagerUserName: fineGrainedManagerUserName,
+            fineGrainedManagerUserSecretManagerKeyARN: fineGrainedManagerUserSecretManagerKeyARN,
+            enforceHTTPS: enforceHTTPS,
+            tlsSecurityPolicy: tlsSecurityPolicy,
+            ebsEnabled: ebsEnabled,
+            ebsIops: ebsIops,
+            ebsVolumeSize: ebsVolumeSize,
+            ebsVolumeType: ebsVolumeType,
+            encryptionAtRestEnabled: encryptionAtRestEnabled,
+            encryptionAtRestKmsKeyARN: encryptionAtRestKmsKeyARN,
+            appLogEnabled: loggingAppLogEnabled,
+            appLogGroup: loggingAppLogGroupARN,
+            nodeToNodeEncryptionEnabled: noneToNodeEncryptionEnabled,
+            vpcId: vpcId,
+            domainRemovalPolicy: domainRemovalPolicy,
+            ...props,
+        });
+
+        this.stacks.push(opensearchStack)
+
+        function getContextForType(optionName: string, expectedType: string): any {
+            const option = scope.node.tryGetContext(optionName)
+
+            // If no context is provided and a default value exists, use it
+            if (option === undefined && defaultValues[optionName]) {
+                return defaultValues[optionName]
+            }
+
+            // Filter out invalid or missing options by setting undefined (empty strings, null, undefined, NaN)
+            if (option !== false && option !== 0 && !option) {
+                return undefined
+            }
+            // Values provided by the CLI will always be represented as a string and need to be parsed
+            if (typeof option === 'string') {
+                if (expectedType === 'number') {
+                    return parseInt(option)
+                }
+                if (expectedType === 'boolean' || expectedType === 'object') {
+                    return JSON.parse(option)
+                }
+            }
+            // Values provided by the cdk.context.json should be of the desired type
+            if (typeof option !== expectedType) {
+                throw new Error(`Type provided by cdk.context.json for ${optionName} was ${typeof option} but expected ${expectedType}`)
+            }
+            return option
+        }
+
+        function parseAccessPolicies(jsonObject: { [x: string]: any; }): PolicyStatement[] {
+            let accessPolicies: PolicyStatement[] = []
+            const statements = jsonObject['Statement']
+            for (let i = 0; i < statements.length; i++) {
+                const statement = PolicyStatement.fromJson(statements[i])
+                accessPolicies.push(statement)
+            }
+            return accessPolicies
+        }
+
+    }
+}