Modify volume structure and checkpoint

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>

deployment/k8/migration-assistant/Chart.yaml

@@ -18,19 +18,14 @@ dependencies:
     version: "0.1.0"
     repository: "file://../services/migration-console"
     condition: migration-console-enabled
-#  - name: replayer
-#    version: "0.1.0"
-#    repository: "file://../services/replayer"
+  - name: replayer
+    version: "0.1.0"
+    repository: "file://../services/replayer"
+    condition: replayer-enabled
   - name: capture-proxy
     version: "0.1.0"
     repository: "file://../services/capture-proxy"
     condition: capture-proxy-enabled
 #  - name: reindex-from-snapshot
 #    version: "0.1.0"
 #    repository: "file://../services/reindex-from-snapshot"
-#  - name: shared-logs-vol
-#    version: "0.1.0"
-#    repository: "file://../services/shared/shared-logs-vol"
-#  - name: snapshot-vol
-#    version: "0.1.0"
-#    repository: "file://../services/shared/snapshot-vol"

deployment/k8/migration-assistant/values.yaml

@@ -1,82 +1,11 @@
 # Define all enabled/disabled migration services
-migration-console-enabled: false
+migration-console-enabled: true
 capture-proxy-enabled: false
-kafka-op-enabled: true
+replayer-enabled: false
+kafka-op-enabled: false
 opensearch-helm-enabled: false
 
-# Define migration service configuration
-migration-console:
-  replicaCount: 1
-  image:
-    repository: migrations/migration_console
-    pullPolicy: IfNotPresent
-    tag: "latest"
-  resources:
-    limits:
-      cpu: 100m
-      memory: 128Mi
-    requests:
-      cpu: 100m
-      memory: 128Mi
-  env:
-    SHARED_LOGS_DIR_PATH: /shared-logs-output/traffic-replayer-default
-  volumes:
-    # For minikube required: minikube mount <local_path>/opensearch-migrations:/opensearch-migrations
-    - name: services-yaml
-      hostPath:
-        path: /opensearch-migrations/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/services.yaml
-        type: File
-    - name: shared-logs
-      persistentVolumeClaim:
-        claimName: shared-logs-pvc
-    - name: snapshot-vol
-      persistentVolumeClaim:
-        claimName: snapshot-vol-pvc
-  volumeMounts:
-    - name: services-yaml
-      mountPath: /etc/migration_services.yaml
-    - name: shared-logs
-      mountPath: /shared-logs-output
-    - name: snapshot-vol
-      mountPath: /storage
-
-capture-proxy:
-  replicaCount: 1
-  image:
-    repository: migrations/capture_proxy
-    pullPolicy: IfNotPresent
-    tag: "latest"
-  service:
-    type: LoadBalancer
-    port: 9201 # Port exposed to external
-    targetPort: 9200 # Container port
-  container:
-    command: >
-      /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy
-      --kafkaConnection kafka-cluster-kafka-bootstrap:9092
-      --destinationUri  https://elasticsearch:19200
-      --insecureDestination
-      --listenPort 9200
-      --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
-      --otelCollectorEndpoint http://localhost:4317
-  resources:
-    limits:
-      cpu: 100m
-      memory: 128Mi
-    requests:
-      cpu: 100m
-      memory: 128Mi
-
-kafka-op:
-  clusterName: kafka-cluster
-  kafkaVersion: 3.8.0
-  kafkaMetadataVersion: 3.8-IV0
-  nodePool:
-    replicas: 1
-    volume:
-      size: 10Gi
-      deleteClaim: true
-
+# Define specific dependency configuration
 opensearch-helm:
   fullnameOverride: "opensearch"
   image:
@@ -90,3 +19,14 @@ opensearch-helm:
       value: myStrongPassword123!
   service:
     type: LoadBalancer
+
+#kafka-op:
+#  clusterName: kafka-cluster
+#  kafkaVersion: 3.8.0
+#  kafkaMetadataVersion: 3.8-IV0
+#  nodePool:
+#    replicas: 1
+#    volume:
+#      size: 10Gi
+#      deleteClaim: true
+

deployment/k8/services/capture-proxy/values.yaml

@@ -1 +1,26 @@
 # Default values for capture proxy.
+replicaCount: 1
+image:
+  repository: migrations/capture_proxy
+  pullPolicy: IfNotPresent
+  tag: "latest"
+service:
+  type: LoadBalancer
+  port: 9201 # Port exposed to external
+  targetPort: 9200 # Container port
+container:
+  command: >
+    /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy
+    --kafkaConnection kafka-cluster-kafka-bootstrap:9092
+    --destinationUri  https://elasticsearch:19200
+    --insecureDestination
+    --listenPort 9200
+    --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
+    --otelCollectorEndpoint http://localhost:4317
+resources:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi

deployment/k8/services/common-lib/templates/_deployment.yaml

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         app: {{ .Chart.Name }}
-      annotations:
-        sidecar.opentelemetry.io/inject: "true" # CORRECT
     spec:
       containers:
         - name: {{ .Chart.Name }}
@@ -29,9 +27,17 @@ spec:
           {{- end }}
           # Load all env variables
           env:
-            {{- range $key, $value := .Values.env }}
-            - name: {{ $key }}
-              value: {{ $value | quote }}
+            {{- range .Values.env }}
+            - name: {{ .name }}
+              {{- if .value }}
+              value: {{ .value }}
+              {{- end }}
+              {{- if .configMapKeyRef }}
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ .configMapKeyRef.name }}
+                  key: {{ .configMapKeyRef.key }}
+              {{- end }}
             {{- end }}
           volumeMounts:
             {{- range .Values.volumeMounts }}

deployment/k8/services/elasticsearch-searchguard/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: elasticsearch-searchguard
+description: A Helm chart to install Elasticsearch with Searchguard
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

deployment/k8/services/elasticsearch-searchguard/values.yaml

@@ -0,0 +1,27 @@
+# Default values for elasticsearch-searchguard.
+replicaCount: 1
+image:
+  repository: migrations/elasticsearch_searchguard
+  pullPolicy: IfNotPresent
+  tag: "latest"
+service:
+  type: LoadBalancer
+  port: 19200 # Port exposed to external
+  targetPort: 9200 # Container port
+resources:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi
+env:
+  - name: path.repo
+    value: "/storage/snapshot"
+volumes:
+  - name: snapshot-vol
+    persistentVolumeClaim:
+      claimName: snapshot-vol-pvc
+volumeMounts:
+  - name: snapshot-vol
+    mountPath: /storage

deployment/k8/services/kafka-op/values.yaml

@@ -1 +1,9 @@
 # Configuration for Strimzi Kafka Operator
+clusterName: kafka-cluster
+kafkaVersion: 3.8.0
+kafkaMetadataVersion: 3.8-IV0
+nodePool:
+  replicas: 1
+  volume:
+    size: 10Gi
+    deleteClaim: true

deployment/k8/services/migration-console/Chart.lock

@@ -2,5 +2,11 @@ dependencies:
 - name: common-lib
   repository: file://../common-lib
   version: 0.1.0
-digest: sha256:20073f8787800aa68dec8f48b8c4ee0c196f0d6ee2eba090164f5a9478995895
-generated: "2024-10-23T18:22:16.76174-05:00"
+- name: snapshot-vol
+  repository: file://../../shared/snapshot-vol
+  version: 0.1.0
+- name: shared-logs-vol
+  repository: file://../../shared/shared-logs-vol
+  version: 0.1.0
+digest: sha256:1860cec80cc6e28129a61bf237e9fde1661dc63cc4ef58534982d23da0b63c75
+generated: "2024-10-25T14:37:35.118738-05:00"

deployment/k8/services/migration-console/Chart.yaml

@@ -8,3 +8,9 @@ dependencies:
   - name: common-lib
     version: "0.1.0"
     repository: "file://../common-lib"
+  - name: snapshot-vol
+    version: "0.1.0"
+    repository: "file://../../shared/snapshot-vol"
+  - name: shared-logs-vol
+    version: "0.1.0"
+    repository: "file://../../shared/shared-logs-vol"

deployment/k8/services/migration-console/migration_services.yaml

@@ -0,0 +1,32 @@
+source_cluster:
+  endpoint: "https://capture-proxy:9201"
+  allow_insecure: true
+  basic_auth:
+    username: "admin"
+    password: "admin"
+target_cluster:
+  endpoint: "https://opensearch-cluster-master:9200"
+  allow_insecure: true
+  basic_auth:
+    username: "admin"
+    password: "myStrongPassword123!"
+metrics_source:
+  prometheus:
+    endpoint: "http://prometheus:9090"
+backfill:
+  reindex_from_snapshot:
+    docker:
+replay:
+  docker:
+snapshot:
+  snapshot_name: "rfs-snapshot"
+  fs:
+    repo_path: "/storage/snapshot"
+  otel_endpoint: "http://localhost:4317"
+metadata_migration:
+  from_snapshot:
+  min_replicas: 0
+  otel_endpoint: "http://localhost:4317"
+kafka:
+  broker_endpoints: "kafka-cluster-kafka-bootstrap:9092"
+  standard: ""

deployment/k8/services/migration-console/templates/configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: services-yaml-config-map
+data:
+  {{- (.Files.Glob "migration_services.yaml").AsConfig | nindent 2 }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-env
+data:
+  STAGE: 'dev'