Address PR Comments

Signed-off-by: Andre Kurait <akurait@amazon.com>

.github/workflows/CI.yml

@@ -200,6 +200,16 @@ jobs:
           node-version: ${{ env.node-version }}
       - name: Install NPM dependencies
         run: npm ci
+      - name: Mock Docker Images for CDK Tests
+        run: docker images &&  docker pull alpine && \
+          docker tag alpine migrations/capture_proxy:latest && \
+          docker tag alpine migrations/capture_proxy_es:latest && \
+          docker tag alpine opensearchproject/opensearch:2 && \
+          docker tag alpine migrations/elasticsearch_searchguard:latest && \
+          docker tag alpine docker.io/apache/kafka:3.7.0 && \
+          docker tag alpine migrations/migration_console:latest && \
+          docker tag alpine migrations/reindex_from_snapshot:latest && \
+          docker tag alpine migrations/traffic_replayer:latest
       - name: Run CDK Jest Tests
         run: npm test
 

TrafficCapture/dockerSolution/build.gradle

@@ -7,10 +7,6 @@ plugins {
 import org.opensearch.migrations.common.CommonUtils
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 
-def calculateDockerHash = { projectName ->
-    CommonUtils.calculateDockerHash(project.fileTree("src/main/docker/${projectName}"))
-}
-
 dependencies {
     constraints {
         implementation('software.amazon.awssdk:secretsmanager:2.25.19') {
@@ -20,17 +16,16 @@ dependencies {
 }
 
 def dockerFilesForExternalServices = [
-        "elasticsearchWithSearchGuard": "elasticsearch_searchguard",
-        "captureProxyBase": "capture_proxy_base",
-        "elasticsearchTestConsole": "elasticsearch_client_test_console",
-        "migrationConsole": "migration_console",
-        "otelCollector": "otel_collector",
+        "elasticsearch_searchguard": "elasticsearchWithSearchGuard",
+        "capture_proxy_base": "captureProxyBase",
+        "elasticsearch_client_test_console": "elasticsearchTestConsole",
+        "migration_console": "migrationConsole",
+        "otel_collector": "otelCollector",
         "grafana": "grafana"
 ]
-
-dockerFilesForExternalServices.each { projectName, dockerImageName ->
+dockerFilesForExternalServices.each { dockerImageName, projectName ->
     def escapedProjectName = projectName;
-    task("buildDockerImage_${escapedProjectName}", type: DockerBuildImage) {
+    task("buildDockerImage_${dockerImageName}", type: DockerBuildImage) {
         if (escapedProjectName == "migrationConsole") {
             def libraries = [
                 project(":libraries:kafkaCommandLineFormatter")
@@ -52,28 +47,28 @@ dockerFilesForExternalServices.each { projectName, dockerImageName ->
     }
 }
 
-static def Sync getMigrationConsoleSyncTask(Project project, String dockerImageName, String escapedProjectName, List<Project> libraries, List<Project> applications) {
+static Sync getMigrationConsoleSyncTask(Project project, String dockerImageName, String escapedProjectName, List<Project> libraries, List<Project> applications) {
      // Create a single sync task to copy the required files
     def destDir = "build/docker/${dockerImageName}_${escapedProjectName}"
     def syncTask = project.tasks.create("syncArtifact_${dockerImageName}_${escapedProjectName}", Sync) {
         into destDir
         duplicatesStrategy = DuplicatesStrategy.EXCLUDE
 
-        // Copy libraries
-        (libraries + applications).each { libProject ->
-            def applicationDestDir = "staging/${libProject.name}/"
-            from (libProject.configurations.findByName("runtimeClasspath").files) {
+        // Applications and Standalone Libraries both have libraries
+        (libraries + applications).each { lib ->
+            def applicationDestDir = "staging/${lib.name}/"
+            from (lib.configurations.findByName("runtimeClasspath").files) {
                 into "${applicationDestDir}/lib"
             }
-            from (libProject.tasks.getByName('jar')) {
+            from (lib.tasks.getByName('jar')) {
                 into "${applicationDestDir}/lib"
             }
         }
 
         // Copy applications
-        applications.each { appProject ->
-            def applicationDestDir = "staging/${appProject.name}/"
-            from (appProject.tasks.getByName('startScripts').outputs.files) {
+        applications.each { app ->
+            def applicationDestDir = "staging/${app.name}/"
+            from (app.tasks.getByName('startScripts').outputs.files) {
                 into "${applicationDestDir}/bin"
             }
         }
@@ -87,7 +82,7 @@ static def Sync getMigrationConsoleSyncTask(Project project, String dockerImageN
     syncTask.dependsOn assembleTasks
 
     // Migration Console base image is the test console
-    syncTask.dependsOn "buildDockerImage_elasticsearchTestConsole"
+    syncTask.dependsOn "buildDockerImage_elasticsearch_client_test_console"
 
     return syncTask
 }
@@ -97,14 +92,14 @@ def javaContainerServices = [
         "capture_proxy_es": ":TrafficCapture:trafficCaptureProxyServer",
         "traffic_replayer": ":TrafficCapture:trafficReplayer"
 ]
-def baseImageProjectOverrides = [
-        "capture_proxy": "captureProxyBase",
-        "capture_proxy_es": "elasticsearchWithSearchGuard",
+def baseImageOverrides = [
+        "capture_proxy": "capture_proxy_base",
+        "capture_proxy_es": "elasticsearch_searchguard",
 ]
 javaContainerServices.each { dockerImageName, projectName ->
     def artifactProject = project(projectName);
     CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project as Project, artifactProject, dockerImageName)
-    CommonUtils.createDockerfile(project, artifactProject, baseImageProjectOverrides[dockerImageName], dockerFilesForExternalServices, dockerImageName)
+    CommonUtils.createDockerfile(project, artifactProject, baseImageOverrides[dockerImageName], dockerFilesForExternalServices, dockerImageName)
 }
 
 javaContainerServices.forEach { dockerImageName, projectName ->
@@ -130,9 +125,9 @@ dockerCompose {
 }
 
 task buildDockerImages {
-    dependsOn buildDockerImage_elasticsearchWithSearchGuard
-    dependsOn buildDockerImage_migrationConsole
-    dependsOn buildDockerImage_otelCollector
+    dependsOn buildDockerImage_elasticsearch_searchguard
+    dependsOn buildDockerImage_migration_console
+    dependsOn buildDockerImage_otel_collector
     dependsOn buildDockerImage_grafana
     dependsOn buildDockerImage_traffic_replayer
     dependsOn buildDockerImage_capture_proxy

TrafficCapture/dockerSolution/src/main/docker/captureProxyBase/Dockerfile

@@ -48,11 +48,7 @@ ENV SEARCHGUARD_CERTS_DIR=$ES_HOME/config
 # The elasticsearch.yml and proxy_tls.yml files are configured to work seamlessly with the Search Guard plugin and TLS settings.
 
 # Add base elasticsearch.yml content
-RUN echo 'searchguard.ssl.transport.pemcert_filepath: esnode.pem' >> $ELASTIC_SEARCH_CONFIG_FILE && \
-    echo 'searchguard.ssl.transport.pemkey_filepath: esnode-key.pem' >> $ELASTIC_SEARCH_CONFIG_FILE && \
-    echo 'searchguard.ssl.transport.pemtrustedcas_filepath: root-ca.pem' >> $ELASTIC_SEARCH_CONFIG_FILE && \
-    echo 'searchguard.ssl.transport.enforce_hostname_verification: false' >> $ELASTIC_SEARCH_CONFIG_FILE && \
-    echo 'searchguard.ssl.http.enabled: true' >> $ELASTIC_SEARCH_CONFIG_FILE && \
+RUN echo 'searchguard.ssl.http.enabled: true' >> $ELASTIC_SEARCH_CONFIG_FILE && \
     echo 'searchguard.ssl.http.pemcert_filepath: esnode.pem' >> $ELASTIC_SEARCH_CONFIG_FILE && \
     echo 'searchguard.ssl.http.pemkey_filepath: esnode-key.pem' >> $ELASTIC_SEARCH_CONFIG_FILE && \
     echo 'searchguard.ssl.http.pemtrustedcas_filepath: root-ca.pem' >> $ELASTIC_SEARCH_CONFIG_FILE

TrafficCapture/trafficCaptureProxyServerTest/build.gradle

@@ -43,12 +43,12 @@ configurations {
 
 
 def dockerFilesForExternalServices = [
-        "nginx": "nginx-perf-test-webserver"
+        "nginx_perf_test_webserver": "nginx"
 ]
 
 // Create the static docker files that aren't hosting migrations java code from this repo
-dockerFilesForExternalServices.each { projectName, dockerImageName ->
-    task("buildDockerImage_${projectName}", type: DockerBuildImage) {
+dockerFilesForExternalServices.each { dockerImageName, projectName ->
+    task("buildDockerImage_${dockerImageName}", type: DockerBuildImage) {
         def hash = calculateDockerHash(projectName)
         images.add("migrations/${dockerImageName}:$hash")
         images.add("migrations/${dockerImageName}:latest")
@@ -60,7 +60,7 @@ def javaContainerServices = [
         "jmeter": ":TrafficCapture:trafficCaptureProxyServerTest"
 ]
 def baseImageProjectOverrides = [
-        "nginx": "nginx-perf-test-webserver"
+        "nginx": "nginx_perf_test_webserver"
 ]
 
 def createContainerTasks = { dockerImageName, projectName ->
@@ -88,7 +88,7 @@ dockerCompose {
 
 task buildDockerImages {
     dependsOn ':TrafficCapture:dockerSolution:buildDockerImage_capture_proxy'
-    dependsOn buildDockerImage_nginx
+    dependsOn buildDockerImage_nginx_perf_test_webserver
     dependsOn buildDockerImage_jmeter
 }
 

TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   webserver:
-    image: 'migrations/nginx-perf-test-webserver:latest'
+    image: 'migrations/nginx_perf_test_webserver:latest'
     networks:
       - testing
     ports:

buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy

@@ -1,5 +1,6 @@
 package org.opensearch.migrations.common
 
+import org.gradle.api.GradleException
 import org.gradle.api.tasks.Sync
 import org.gradle.api.Project
 import com.bmuschko.gradle.docker.tasks.image.Dockerfile
@@ -54,10 +55,16 @@ class CommonUtils {
             destFile = dockerBuildProject.file("${dockerBuildDir}/Dockerfile")
             dependsOn "copyArtifact_${dockerImageName}"
             if (baseImageProjectOverride) {
-                def dependentDockerImageName = dockerFilesForExternalServices.get(baseImageProjectOverride)
-                def hashNonce = CommonUtils.calculateDockerHash(
-                        dockerBuildProject.fileTree("src/main/docker/${baseImageProjectOverride}"))
-                from "migrations/${dependentDockerImageName}:${hashNonce}"
+                def dependentDockerImageProjectName = dockerFilesForExternalServices.get(baseImageProjectOverride)
+                if (dependentDockerImageProjectName == null) {
+                    throw new GradleException("Unexpected baseImageOverride " + baseImageProjectOverride)
+                }
+                def dockerFileTree = dockerBuildProject.fileTree("src/main/docker/${dependentDockerImageProjectName}")
+                if (!dockerFileTree.files) {
+                    throw new GradleException("File tree for ${dependentDockerImageProjectName} does not exist or is empty")
+                }
+                def hashNonce = CommonUtils.calculateDockerHash(dockerFileTree)
+                from "migrations/${baseImageProjectOverride}:${hashNonce}"
                 def dependencyName = "buildDockerImage_${baseImageProjectOverride}";
                 dependsOn dependencyName
                 if (baseImageProjectOverride.startsWith("elasticsearch")) {

deployment/cdk/opensearch-service-migration/cdk.context.json

@@ -1,31 +1,33 @@
 {
   "default": {
-    "stage": "test-stage",
+    "stage": "<STAGE>",
     "targetCluster": {
-      "endpoint": "https://target-cluster.com:443",
+      "endpoint": "<TARGET_CLUSTER_ENDPOINT>",
       "auth": {
-        "type": "none"
+        "type": "none | basic | sigv4",
+        "// basic auth documentation": "The next two lines are releavant for basic auth only",
+        "username": "<USERNAME>",
+        "passwordFromSecretArn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
+        "// sigv4 documentation": "The next two lines are releavant for sigv4 only",
+        "region": "<REGION>",
+        "serviceSigningName": "es | aoss"
       }
     },
     "sourceCluster": {
-      "endpoint": "https://source-cluster.com:443",
+      "endpoint": "<SOURCE_CLUSTER_ENDPOINT>",
       "auth": {
-        "type": "none"
+        "type": "none | basic | sigv4",
+        "// basic auth documentation": "The next two lines are releavant for basic auth only",
+        "username": "<USERNAME>",
+        "passwordFromSecretArn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
+        "// sigv4 documentation": "The next two lines are releavant for sigv4 only",
+        "region": "<REGION>",
+        "serviceSigningName": "es | aoss"
       }
     },
+    "vpcId": "<VPC_ID>",
     "reindexFromSnapshotServiceEnabled": true,
     "reindexFromSnapshotMaxShardSizeGiB": 80,
-    "migrationAssistanceEnabled": true,
-    "migrationConsoleServiceEnabled": true,
-    "otelCollectorEnabled": true,
-    "targetClusterProxyServiceEnabled": true
-  },
-  "availability-zones:account=767398060394:region=us-east-1": [
-    "us-east-1a",
-    "us-east-1b",
-    "us-east-1c",
-    "us-east-1d",
-    "us-east-1e",
-    "us-east-1f"
-  ]
-}
+    "trafficReplayerServiceEnabled": true
+  }
+}

deployment/cdk/opensearch-service-migration/lib/common-utilities.ts

@@ -440,26 +440,29 @@ export function isRegionGovCloud(region: string): boolean {
     return region.startsWith('us-gov-');
 }
 
+
 /**
- * Creates a Docker image asset from the specified image name.
+ * Creates a Local Docker image asset from the specified image name.
  *
- * This function creates a Docker image asset from the specified image name.
- * It can use a local or remote image.
+ * This allows us to create a private ECR repo for any image allowing us to have a consistent
+ * experience across VPCs and regions (e.g. running within VPC in gov-cloud with no internet access)
+ * 
+ * This works by creating a temp Dockerfile with only the FROM with the param imageName and
+ * using that Dockerfile with cdk.assets to create a local Docker image asset.
  *
- * @param {string} imageName - The name of the Docker image to use as the base image.
+ * @param {string} imageName - The name of the Docker image to save as a tarball and use in CDK.
  * @returns {ContainerImage} - A `ContainerImage` object representing the Docker image asset.
  */
-export function makeDockerImageAsset(scope: Construct, serviceName: string, imageName: string): ContainerImage {
+export function makeLocalAssetContainerImage(scope: Construct, imageName: string): ContainerImage {
     const sanitizedImageName = imageName.replace(/[^a-zA-Z0-9-_]/g, '_');
     const tempDir = mkdtempSync(join(tmpdir(), 'docker-build-' + sanitizedImageName));
     const dockerfilePath = join(tempDir, 'Dockerfile');
     const dockerfileContent = `
         FROM ${imageName}
     `;
     writeFileSync(dockerfilePath, dockerfileContent);
-    const asset = new DockerImageAsset(scope, serviceName + 'Image', {
+    const asset = new DockerImageAsset(scope, 'ServiceImage', {
         directory: tempDir,
     });
     return ContainerImage.fromDockerImageAsset(asset);
-}
-
+}

deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts

@@ -81,7 +81,7 @@ export class CaptureProxyESStack extends MigrationServiceCore {
 
         this.createService({
             serviceName: "capture-proxy-es",
-            dockerImageRegistryName: "migrations/capture_proxy_es:latest",
+            dockerImageName: "migrations/capture_proxy_es:latest",
             dockerImageCommand: ['/bin/sh', '-c', command.concat(" & wait -n 1")],
             securityGroups: securityGroups,
             taskRolePolicies: servicePolicies,

deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts

@@ -140,7 +140,7 @@ export class CaptureProxyStack extends MigrationServiceCore {
 
         this.createService({
             serviceName: serviceName,
-            dockerImageRegistryName: "migrations/capture_proxy:latest",
+            dockerImageName: "migrations/capture_proxy:latest",
             dockerImageCommand: ['/bin/sh', '-c', command],
             securityGroups: securityGroups,
             taskRolePolicies: servicePolicies,

deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts

@@ -39,7 +39,7 @@ export class ElasticsearchStack extends MigrationServiceCore {
 
         this.createService({
             serviceName: "elasticsearch",
-            dockerImageRegistryName: "migrations/elasticsearch_searchguard:latest",
+            dockerImageName: "migrations/elasticsearch_searchguard:latest",
             securityGroups: securityGroups,
             portMappings: [servicePort],
             cpuArchitecture: props.fargateCpuArch,

deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-stack.ts

@@ -42,7 +42,7 @@ export class KafkaStack extends MigrationServiceCore {
         });
         this.createService({
             serviceName: "kafka",
-            dockerImageRegistryName: "docker.io/apache/kafka:3.7.0",
+            dockerImageName: "docker.io/apache/kafka:3.7.0",
             securityGroups: securityGroups,
             // see https://github.com/apache/kafka/blob/3.7/docker/examples/jvm/single-node/plaintext/docker-compose.yml
             environment: {

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts

@@ -313,7 +313,7 @@ export class MigrationConsoleStack extends MigrationServiceCore {
 
         this.createService({
             serviceName: "migration-console",
-            dockerImageRegistryName: "migrations/migration_console:latest",
+            dockerImageName: "migrations/migration_console:latest",
             securityGroups: securityGroups,
             portMappings: servicePortMappings,
             dockerImageCommand: imageCommand,

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts

@@ -18,7 +18,7 @@ import {
 import {Duration, RemovalPolicy, Stack} from "aws-cdk-lib";
 import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs";
 import {PolicyStatement, Role} from "aws-cdk-lib/aws-iam";
-import {createDefaultECSTaskRole, makeDockerImageAsset} from "../common-utilities";
+import {createDefaultECSTaskRole, makeLocalAssetContainerImage} from "../common-utilities";
 import {OtelCollectorSidecar} from "./migration-otel-collector-sidecar";
 import { IApplicationTargetGroup, INetworkTargetGroup } from "aws-cdk-lib/aws-elasticloadbalancingv2";
 
@@ -28,7 +28,7 @@ export interface MigrationServiceCoreProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly securityGroups: ISecurityGroup[],
     readonly cpuArchitecture: CpuArchitecture,
-    readonly dockerImageRegistryName: string,
+    readonly dockerImageName: string,
     readonly dockerImageCommand?: string[],
     readonly taskRolePolicies?: PolicyStatement[],
     readonly mountPoints?: MountPoint[],
@@ -76,7 +76,7 @@ export class MigrationServiceCore extends Stack {
             props.volumes.forEach(vol => serviceTaskDef.addVolume(vol))
         }
 
-        const serviceImage = makeDockerImageAsset(this, props.serviceName, props.dockerImageRegistryName)
+        const serviceImage = makeLocalAssetContainerImage(this, props.dockerImageName)
 
         const serviceLogGroup = new LogGroup(this, 'ServiceLogGroup',  {
             retention: RetentionDays.ONE_MONTH,

deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts

@@ -83,7 +83,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
 
         this.createService({
             serviceName: dnsNameForContainer,
-            dockerImageRegistryName: "opensearchproject/opensearch:2",
+            dockerImageName: "opensearchproject/opensearch:2",
             securityGroups: securityGroups,
             environment: {
                 "cluster.name": "os-docker-cluster",

deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts

@@ -167,7 +167,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
         this.createService({
             serviceName: 'reindex-from-snapshot',
             taskInstanceCount: 0,
-            dockerImageRegistryName: "migrations/reindex_from_snapshot:latest",
+            dockerImageName: "migrations/reindex_from_snapshot:latest",
             dockerImageCommand: ['/bin/sh', '-c', "/rfs-app/entrypoint.sh"],
             securityGroups: securityGroups,
             volumes: volumes,