Initial attempt for full E2E test

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
lewijacn · Oct 22, 2024 · f2b7f64 · f2b7f64
1 parent db0075e
commit f2b7f64
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 1 deletion.
diff --git a/...le/lib/integ_test/integ_test/e2e_tests.py → ...e/lib/integ_test/integ_test/full_tests.py b/...le/lib/integ_test/integ_test/e2e_tests.py → ...e/lib/integ_test/integ_test/full_tests.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import pytest
 import unittest
 from http import HTTPStatus
@@ -25,6 +26,11 @@ def initialize(request):
     pytest.unique_id = unique_id
     source_cluster = pytest.console_env.source_cluster
     target_cluster = pytest.console_env.target_cluster
+    # If in AWS, modify source and target objects here to route requests through the created ALB to verify its operation
+    if 'AWS_EXECUTION_ENV' in os.environ:
+        logger.info("Detected an AWS environment")
+        stage = request.config.getoption("--stage")
+        source_cluster.endpoint = f"https://alb.migration.{stage}.local:9201"
     backfill: Backfill = pytest.console_env.backfill
     assert backfill is not None
     metadata: Metadata = pytest.console_env.metadata
@@ -90,7 +96,7 @@ def test_e2e_0001_default(self):
         snapshot.create(wait=True)
         metadata.migrate()
         backfill.start()
-        backfill.scale(units=10)
+        backfill.scale(units=2)
         # This document was created after snapshot and should not be included in Backfill but expected in Replay
         create_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id_base + "_2",
                         expected_status_code=HTTPStatus.CREATED, test_case=self)

diff --git a/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy b/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy
@@ -0,0 +1,9 @@
+def gitBranch = params.GIT_BRANCH ?: 'main'
+def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
+
+library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM(
+        [$class: 'GitSCMSource',
+         remote: "${gitUrl}"])
+
+// Shared library function (location from root: vars/fullDefaultE2ETest.groovy)
+fullDefaultE2ETest()
diff --git a/vars/defaultIntegPipeline.groovy b/vars/defaultIntegPipeline.groovy
@@ -159,6 +159,7 @@ def call(Map config = [:]) {
                                     def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " +
                                             "--junitxml=${test_result_file} ${integTestCommand} " +
                                             "--unique_id ${uniqueId} " +
+                                            "--stage ${stage}" +
                                             "-s"
                                     withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) {
                                         withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", duration: 3600, roleSessionName: 'jenkins-session') {

diff --git a/vars/fullDefaultE2ETest.groovy b/vars/fullDefaultE2ETest.groovy
@@ -0,0 +1,73 @@
+
+def call(Map config = [:]) {
+    def sourceContextId = 'source-single-node-ec2'
+    def migrationContextId = 'full-migration'
+    def source_cdk_context = """
+        {
+          "source-single-node-ec2": {
+            "suffix": "ec2-source-<STAGE>",
+            "networkStackSuffix": "ec2-source-<STAGE>",
+            "distVersion": "6.8.23",
+            "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-6.8.23.tar.gz",
+            "captureProxyEnabled": false,
+            "securityDisabled": true,
+            "minDistribution": false,
+            "cpuArch": "x64",
+            "isInternal": true,
+            "singleNodeCluster": true,
+            "networkAvailabilityZones": 2,
+            "dataNodeCount": 1,
+            "managerNodeCount": 0,
+            "serverAccessType": "ipv4",
+            "restrictServerAccessTo": "0.0.0.0/0"
+          }
+        }
+    """
+    def migration_cdk_context = """
+        {
+          "full-migration": {
+            "stage": "<STAGE>",
+            "vpcId": "<VPC_ID>",
+            "engineVersion": "OS_2.11",
+            "domainName": "os-cluster-<STAGE>",
+            "dataNodeCount": 2,
+            "openAccessPolicyEnabled": true,
+            "domainRemovalPolicy": "DESTROY",
+            "artifactBucketRemovalPolicy": "DESTROY",
+            "captureProxyServiceEnabled": true,
+            "targetClusterProxyServiceEnabled": true,
+            "trafficReplayerServiceEnabled": true,
+            "trafficReplayerExtraArgs": "--speedup-factor 10.0",
+            "reindexFromSnapshotServiceEnabled": true,
+            "sourceCluster": {
+                "endpoint": "<SOURCE_CLUSTER_ENDPOINT>",
+                "auth": {"type": "none"},
+                "version": "ES_6.8.23"
+            },
+            "tlsSecurityPolicy": "TLS_1_2",
+            "enforceHTTPS": true,
+            "nodeToNodeEncryptionEnabled": true,
+            "encryptionAtRestEnabled": true,
+            "vpcEnabled": true,
+            "vpcAZCount": 2,
+            "domainAZCount": 2,
+            "mskAZCount": 2,
+            "migrationAssistanceEnabled": true,
+            "replayerOutputEFSRemovalPolicy": "DESTROY",
+            "migrationConsoleServiceEnabled": true,
+            "otelCollectorEnabled": true
+          }
+        }
+    """
+
+    defaultIntegPipeline(
+            sourceContext: source_cdk_context,
+            migrationContext: migration_cdk_context,
+            sourceContextId: sourceContextId,
+            migrationContextId: migrationContextId,
+            defaultStageId: 'full-integ',
+            skipCaptureProxyOnNodeSetup: true,
+            jobName: 'full-default-e2e-test',
+            integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py'
+    )
+}