Merge pull request #2 from AndreKurait/requireSourceVersionInCdkSynth

opensearch-project · Oct 17, 2024 · 579e178 · 579e178
2 parents c92da77 + a4973c4
commit 579e178
Show file tree

Hide file tree

Showing 11 changed files with 50 additions and 25 deletions.
diff --git a/deployment/cdk/opensearch-service-migration/cdk.context.json b/deployment/cdk/opensearch-service-migration/cdk.context.json
@@ -15,7 +15,7 @@
     },
     "sourceCluster": {
       "endpoint": "<SOURCE_CLUSTER_ENDPOINT>",
-      "version": "<SOURCE_CLUSTER_VERSION, ES_7_10 or OS_1_3>",
+      "version": "<SOURCE_CLUSTER_VERSION, ES_7.10 or OS_1.3>",
       "auth": {
         "type": "none | basic | sigv4",
         "// basic auth documentation": "The next two lines are releavant for basic auth only",

diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts
@@ -5,6 +5,7 @@ import {RemovalPolicy, Stack} from "aws-cdk-lib";
 import { IStringParameter, StringParameter } from "aws-cdk-lib/aws-ssm";
 import * as forge from 'node-forge';
 import { ClusterYaml } from "./migration-services-yaml";
+import { CdkLogger } from "./cdk-logger";
 
 export function getSecretAccessPolicy(secretArn: string): PolicyStatement {
     return new PolicyStatement({

diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
@@ -222,14 +222,21 @@ export class StackComposer {
         if (!sourceClusterDefinition && (sourceClusterEndpointField || sourceClusterDisabledField)) {
             CdkLogger.warn("`sourceClusterDisabled` and `sourceClusterEndpoint` are being deprecated in favor of a `sourceCluster` object.")
             CdkLogger.warn("Please update your CDK context block to use the `sourceCluster` object.")
+            CdkLogger.warn("Defaulting to source cluster version: ES_7.10")
             sourceClusterDefinition = {
                 "disabled": sourceClusterDisabledField,
                 "endpoint": sourceClusterEndpointField,
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             }
         }
         const sourceClusterDisabled = !!sourceClusterDefinition?.disabled
         const sourceCluster = (sourceClusterDefinition && !sourceClusterDisabled) ? parseClusterDefinition(sourceClusterDefinition) : undefined
+        if (sourceCluster) {
+            if (!sourceCluster.version) {
+                throw new Error("The `sourceCluster` object requires a `version` field.")
+            }
+        }
         const sourceClusterEndpoint = sourceCluster?.endpoint
 
         if (managedServiceSourceSnapshotEnabled && !sourceCluster?.auth.sigv4) {

diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts
@@ -245,7 +245,7 @@ describe('validateFargateCpuArch', () => {
     test('parseClusterDefinition with basic auth parameters', () => {
         const clusterDefinition = {
             endpoint: 'https://target-cluster',
-            version: 'ES 7.10',
+            version: 'ES_7.10',
             auth: {
               type: 'basic',
               username: 'admin',

diff --git a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts
@@ -126,7 +126,8 @@ describe('Migration Services YAML Tests', () => {
             migrationConsoleServiceEnabled: true,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             },
             reindexFromSnapshotServiceEnabled: true,
             trafficReplayerServiceEnabled: true,
@@ -178,7 +179,8 @@ describe('Migration Services YAML Tests', () => {
             migrationConsoleServiceEnabled: true,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             },
             targetCluster: {
                 "endpoint": "https://target-cluster",
@@ -231,7 +233,8 @@ describe('Migration Services YAML Tests', () => {
             migrationConsoleServiceEnabled: true,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             },
             targetCluster: {
                 "endpoint": "https://target-cluster",

diff --git a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
@@ -63,7 +63,8 @@ describe('NetworkStack Tests', () => {
             vpcAZCount: 2,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             }
         }
 
@@ -89,7 +90,8 @@ describe('NetworkStack Tests', () => {
             vpcAZCount: 2,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             }
         }
 
@@ -108,7 +110,8 @@ describe('NetworkStack Tests', () => {
             vpcAZCount: 2,
             sourceCluster: {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             }
         }
 

diff --git a/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts
@@ -59,7 +59,8 @@ describe('OpenSearch Domain Stack Tests', () => {
       domainRemovalPolicy: "DESTROY",
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       }
     }
 
@@ -115,7 +116,8 @@ describe('OpenSearch Domain Stack Tests', () => {
       domainRemovalPolicy: "DESTROY",
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       }
     }
 
@@ -151,7 +153,8 @@ describe('OpenSearch Domain Stack Tests', () => {
       nodeToNodeEncryptionEnabled: true,
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       }
     }
 
@@ -174,7 +177,8 @@ describe('OpenSearch Domain Stack Tests', () => {
       nodeToNodeEncryptionEnabled: "true",
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       }
     }
 

diff --git a/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts
@@ -40,7 +40,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
       vpcEnabled: true,
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       },
       reindexFromSnapshotServiceEnabled: true,
       stage: 'unit-test',
@@ -80,7 +81,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
       vpcEnabled: true,
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       },
       reindexFromSnapshotServiceEnabled: true,
       stage: 'unit-test',
@@ -119,7 +121,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
               {
                 "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
               },
-              " --max-shard-size-bytes 85899345920"
+              " --max-shard-size-bytes 85899345920 --source-version \"ES_7.10\""
             ],
           ],
         }
@@ -150,7 +152,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
       stage: 'unit-test',
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       },
       targetCluster: {
         "endpoint": "https://target-cluster",
@@ -187,7 +190,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
               {
                 "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
               },
-              " --max-shard-size-bytes 85899345920 --target-aws-service-signing-name aoss --target-aws-region eu-west-1",
+              " --max-shard-size-bytes 85899345920 --target-aws-service-signing-name aoss --target-aws-region eu-west-1 --source-version \"ES_7.10\"",
             ],
           ],
         }
@@ -218,7 +221,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
       stage: 'unit-test',
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       },
       migrationAssistanceEnabled: true,
     };
@@ -243,7 +247,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
       stage: 'unit-test',
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       },
       reindexFromSnapshotExtraArgs: '--custom-arg value --flag --snapshot-name \"custom-snapshot\"',
       migrationAssistanceEnabled: true,
@@ -276,7 +281,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
               {
                 "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
               },
-              " --max-shard-size-bytes 85899345920 --custom-arg value --flag --snapshot-name \"custom-snapshot\""
+              " --max-shard-size-bytes 85899345920 --source-version \"ES_7.10\" --custom-arg value --flag --snapshot-name \"custom-snapshot\""
             ]
           ]
         }

diff --git a/deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json b/deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json
@@ -7,7 +7,7 @@
     "otelCollectorEnabled": false,
     "sourceCluster": {
       "endpoint": "https://test-cluster",
-      "version": "ES 7.10",
+      "version": "ES_7.10",
       "auth": { "type": "none" }
     }
   }

diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts
@@ -118,7 +118,8 @@ describe('Stack Composer Ordering Tests', () => {
             "reindexFromSnapshotServiceEnabled": false,
             "sourceCluster": {
                 "endpoint": "https://test-cluster",
-                "auth": {"type": "none"}
+                "auth": {"type": "none"},
+                "version": "ES_7.10"
             }
         }
 

diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
@@ -43,7 +43,7 @@ describe('Stack Composer Tests', () => {
     expect(createStackFunc).toThrow()
   })
 
-  test('Test ES 7.10 engine version format is parsed', () => {
+  test('Test ES_7.10 engine version format is parsed', () => {
     const contextOptions = {
       engineVersion: "ES_7.10"
     }
@@ -219,7 +219,8 @@ describe('Stack Composer Tests', () => {
       migrationConsoleServiceEnabled: true,
       sourceCluster: {
         "endpoint": "https://test-cluster",
-        "auth": {"type": "none"}
+        "auth": {"type": "none"},
+        "version": "ES_7.10"
       }
     }