diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 5bfb3a4a43..101d09d031 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -23,11 +23,9 @@ pipeline { } triggers { parameterizedCron ''' - H */6 * * * %INPUT_MANIFEST=1.3.18/opensearch-dashboards-1.3.18.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.15.1/opensearch-2.15.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-dashboards-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H */6 * * * %INPUT_MANIFEST=1.3.18/opensearch-1.3.18.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip ''' diff --git a/jenkins/opensearch/benchmark-pull-request.jenkinsfile b/jenkins/opensearch/benchmark-pull-request.jenkinsfile new file mode 100644 index 0000000000..88441c49d1 --- /dev/null +++ b/jenkins/opensearch/benchmark-pull-request.jenkinsfile @@ -0,0 +1,296 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +lib = library(identifier: 'jenkins@6.4.3', retriever: modernSCM([ + + $class: 'GitSCMSource', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', +])) + +pipeline { + agent {label 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test'} + options { + timeout(time: 24, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) + } + environment { + JOB_NAME = 'pull-request-benchmark-test' + } + parameters { + string( + name: 'DISTRIBUTION_URL', + description: 'Publicly available download url of the OpenSearch artifact tarball. Currently only supports x64 arch.', + ) + string( + name: 'DISTRIBUTION_VERSION', + description: 'The distribution version of of the OpenSearch artifact, only to be provided in combination with DISTRIBUTION_URL param.', + trim: true + ) + booleanParam( + name: 'SECURITY_ENABLED', + description: 'Mention if the cluster is secured or insecured.', + defaultValue: false, + ) + booleanParam( + name: 'SINGLE_NODE_CLUSTER', + description: 'Benchmark test on a single node cluster', + defaultValue: true + ) + booleanParam( + name: 'MIN_DISTRIBUTION', + description: 'Use OpenSearch min distribution', + defaultValue: false + ) + string( + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads.', + defaultValue: 'nyc_taxis', + trim: true + ) + string( + name: 'MANAGER_NODE_COUNT', + description: 'Number of cluster manager nodes, empty value defaults to 3.', + trim: true + ) + string( + name: 'DATA_NODE_COUNT', + description: 'Number of cluster data nodes, empty value defaults to 2.', + trim: true + ) + string( + name: 'CLIENT_NODE_COUNT', + description: 'Number of cluster client nodes, empty value default to 0.', + trim: true + ) + string( + name: 'INGEST_NODE_COUNT', + description: 'Number of cluster INGEST nodes, empty value defaults to 0.', + trim: true + ) + string( + name: 'ML_NODE_COUNT', + description: 'Number of cluster ml nodes, empty value defaults to 0.', + trim: true + ) + string( + name: 'DATA_INSTANCE_TYPE', + description: 'EC2 instance type for data node, empty defaults to r5.xlarge.', + trim: true + ) + string( + name: 'DATA_NODE_STORAGE', + description: 'Data node ebs block storage size, empty value defaults to 100Gb', + trim: true + ) + string( + name: 'ML_NODE_STORAGE', + description: 'ML node ebs block storage size, empty value defaults to 100Gb', + trim: true + ) + string( + name: 'JVM_SYS_PROPS', + description: 'A comma-separated list of key=value pairs that will be added to jvm.options as JVM system properties.', + trim: true + ) + string( + name: 'ADDITIONAL_CONFIG', + description: 'Additional opensearch.yml config parameters passed as JSON. e.g., `opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT`', + trim: true + ) + booleanParam( + name: 'USE_50_PERCENT_HEAP', + description: 'Use 50 percent of physical memory as heap.', + defaultValue: true + ) + string( + name: 'USER_TAGS', + description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ', + trim: true + ) + string( + name: 'WORKLOAD_PARAMS', + description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`', + trim: true + ) + string( + name: 'TEST_PROCEDURE', + description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`', + trim: true + ) + string( + name: 'EXCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + string( + name: 'INCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + booleanParam( + name: 'CAPTURE_NODE_STAT', + description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.', + defaultValue: false + ) + string( + name: 'TELEMETRY_PARAMS', + description: 'Allows to set parameters for telemetry devices. Use json type. e.g.,{"node-stats-include-indices":"true","node-stats-include-indices-metrics":"segments"}', + trim: true + ) + } + triggers { + GenericTrigger( + genericVariables: [ + [key: 'pull_request_number', value: '$.pull_request_number'], + [key: 'repository', value: '$.repository'], + [key: 'DISTRIBUTION_URL', value: '$.DISTRIBUTION_URL'], + [key: 'DISTRIBUTION_VERSION', value: '$.DISTRIBUTION_VERSION'], + [key: 'TEST_WORKLOAD', value: '$.TEST_WORKLOAD'], + [key: 'SECURITY_ENABLED', value: '$.SECURITY_ENABLED'], + [key: 'SINGLE_NODE_CLUSTER', value: '$.SINGLE_NODE_CLUSTER'], + [key: 'MIN_DISTRIBUTION', value: '$.MIN_DISTRIBUTION'], + [key: 'MANAGER_NODE_COUNT', value: '$.MANAGER_NODE_COUNT'], + [key: 'DATA_NODE_COUNT', value: '$.DATA_NODE_COUNT'], + [key: 'CLIENT_NODE_COUNT', value: '$.CLIENT_NODE_COUNT'], + [key: 'INGEST_NODE_COUNT', value: '$.INGEST_NODE_COUNT'], + [key: 'ML_NODE_COUNT', value: '$.ML_NODE_COUNT'], + [key: 'DATA_INSTANCE_TYPE', value: '$.DATA_INSTANCE_TYPE'], + [key: 'DATA_NODE_STORAGE', value: '$.DATA_NODE_STORAGE'], + [key: 'ML_NODE_STORAGE', value: '$.ML_NODE_STORAGE'], + [key: 'JVM_SYS_PROPS', value: '$.JVM_SYS_PROPS'], + [key: 'ADDITIONAL_CONFIG', value: '$.ADDITIONAL_CONFIG'], + [key: 'USE_50_PERCENT_HEAP', value: '$.USE_50_PERCENT_HEAP'], + [key: 'USER_TAGS', value: '$.USER_TAGS'], + [key: 'WORKLOAD_PARAMS', value: '$.WORKLOAD_PARAMS'], + [key: 'TEST_PROCEDURE', value: '$.TEST_PROCEDURE'], + [key: 'EXCLUDE_TASKS', value: '$.EXCLUDE_TASKS'], + [key: 'INCLUDE_TASKS', value: '$.INCLUDE_TASKS'], + [key: 'CAPTURE_NODE_STAT', value: '$.CAPTURE_NODE_STAT'], + [key: 'TELEMETRY_PARAMS', value: '$.TELEMETRY_PARAMS'] + ], + tokenCredentialId: 'jenkins-pr-benchmark-generic-webhook-token', + causeString: 'Triggered by comment on PR on OpenSearch core repository', + printContributedVariables: true, + printPostContent: true + ) + } + + stages { + stage('validate-and-set-parameters') { + steps { + script { + if (DISTRIBUTION_URL == '' || DISTRIBUTION_VERSION == '') { + currentBuild.result = 'ABORTED' + error("Benchmark Tests failed to start. Provide DISTRIBUTION_URL and DISTRIBUTION_VERSION to run tests") + } + env.ARCHITECTURE = "x64" + lib.jenkins.Messages.new(this).add(JOB_NAME, "Benchmark tests for ${DISTRIBUTION_URL}") + if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { + currentBuild.description = "Benchmark initiated by PR:${pull_request_number} on ${repository}" + } + else { + currentBuild.description = "Running benchmark test for distribution-url: ${DISTRIBUTION_URL} distribution-version: ${DISTRIBUTION_VERSION}" + } + } + } + } + stage('benchmark-pull-request') { + steps { + script { + echo "security-enabled: ${SECURITY_ENABLED}" + + runBenchmarkTestScript( + distributionUrl: DISTRIBUTION_URL, + distributionVersion: DISTRIBUTION_VERSION, + workload: TEST_WORKLOAD, + insecure: !(params.SECURITY_ENABLED), + singleNode: SINGLE_NODE_CLUSTER, + minDistribution: MIN_DISTRIBUTION, + use50PercentHeap: USE_50_PERCENT_HEAP, + managerNodeCount: MANAGER_NODE_COUNT, + dataNodeCount: DATA_NODE_COUNT, + clientNodeCount: CLIENT_NODE_COUNT, + ingestNodeCount: INGEST_NODE_COUNT, + mlNodeCount: ML_NODE_COUNT, + dataInstanceType: DATA_INSTANCE_TYPE, + additionalConfig: ADDITIONAL_CONFIG, + dataStorageSize: DATA_NODE_STORAGE, + mlStorageSize: ML_NODE_STORAGE, + jvmSysProps: JVM_SYS_PROPS, + userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}", + suffix: "${BUILD_NUMBER}", + workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, + captureNodeStat: CAPTURE_NODE_STAT, + telemetryParams: TELEMETRY_PARAMS + ) + + sh "cp /tmp/final_result_${BUILD_NUMBER}.md final_result_${BUILD_NUMBER}.md" + } + } + post { + success { + sh "sed -i '1i\\#### Benchmark Results for Job: ${BUILD_URL}' final_result_${BUILD_NUMBER}.md" + script { + if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def pull_request = Integer.parseInt("${pull_request_number}") + sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md") + } + } + } + postCleanup() + + } + failure { + script { + if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def pull_request = Integer.parseInt("${pull_request_number}") + sh ("gh pr comment ${pull_request} --repo ${repository} --body The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.") + } + } + } + + } + aborted { + script { + def stackNames = [ + "opensearch-infra-stack-${BUILD_NUMBER}" + ] + withCredentials([string(credentialsId: 'perf-test-account-id', variable: 'PERF_TEST_ACCOUNT_ID')]) { + withAWS(role: 'cfn-set-up', roleAccount: "${PERF_TEST_ACCOUNT_ID}", duration: 900, roleSessionName: 'jenkins-session', region: 'us-east-1') { + try { + for (String stackName : stackNames) { + def stack = null + try { + stack = cfnDescribe(stack: stackName) + } catch (Exception) { + echo "Stack '${stackName}' does not exist, nothing to remove" + } + if (stack != null) { + echo "Deleting stack '${stackName}'" + cfnDelete(stack: stackName, pollInterval:1000) + } + } + } catch (Exception e) { + error "Exception occurred while deleting the CloudFormation stack: ${e.toString()}" + } + } + } + } + postCleanup() + + } + } + } + } +} diff --git a/jenkins/opensearch/benchmark-test.jenkinsfile b/jenkins/opensearch/benchmark-test.jenkinsfile index 30fee3c359..0767b14105 100644 --- a/jenkins/opensearch/benchmark-test.jenkinsfile +++ b/jenkins/opensearch/benchmark-test.jenkinsfile @@ -65,6 +65,8 @@ pipeline { H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.11.1;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-211","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-211","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.7.0/opensearch-2.7.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.7.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-270","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-270","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.12.0/opensearch-2.12.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.12.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.13.0/opensearch-2.13.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.13.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.14.0/opensearch-2.14.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.14.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true diff --git a/manifests/1.3.18/opensearch-1.3.18.yml b/manifests/1.3.18/opensearch-1.3.18.yml index 85be619920..5480a24f0e 100644 --- a/manifests/1.3.18/opensearch-1.3.18.yml +++ b/manifests/1.3.18/opensearch-1.3.18.yml @@ -10,13 +10,13 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: '1.3' + ref: 482ebc75b9ec04a4de5785c6e015144a2bb1a6c3 checks: - gradle:publish - gradle:properties:version - name: common-utils repository: https://github.com/opensearch-project/common-utils.git - ref: '1.3' + ref: 52d56e027bbaba4535b3fc18b2995089d9ec7628 checks: - gradle:publish - gradle:properties:version @@ -25,7 +25,7 @@ components: - windows - name: job-scheduler repository: https://github.com/opensearch-project/job-scheduler.git - ref: '1.3' + ref: 54639fbe22d1ff2a9ea7261ec3e5dc41aa81369e checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -34,7 +34,7 @@ components: - windows - name: security repository: https://github.com/opensearch-project/security.git - ref: '1.3' + ref: c9cf6b89d554889e771fe2ebe007889b7a0f85cf checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -43,7 +43,7 @@ components: - windows - name: ml-commons repository: https://github.com/opensearch-project/ml-commons.git - ref: '1.3' + ref: 0cd33e31dce0a59f420c223d34215bde73c1557a checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin @@ -54,7 +54,7 @@ components: - common-utils - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git - ref: '1.3' + ref: 4415472663d99281ebf61e8e2e179edd0e5e5555 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -62,7 +62,7 @@ components: - linux - name: alerting repository: https://github.com/opensearch-project/alerting.git - ref: '1.3' + ref: c9dbabd1f344d8acc3ada76e0f208b81d6d1e8d4 checks: - gradle:properties:version - gradle:dependencies:opensearch.version: alerting @@ -73,7 +73,7 @@ components: - common-utils - name: opensearch-reports repository: https://github.com/opensearch-project/reporting.git - ref: '1.3' + ref: 22135bba4ee38886493c87dd8cda68252f65408f checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -85,7 +85,7 @@ components: - job-scheduler - name: cross-cluster-replication repository: https://github.com/opensearch-project/cross-cluster-replication.git - ref: '1.3' + ref: 7bf26261b3296efca3bb1a6899265f2b41e773b7 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -96,7 +96,7 @@ components: - common-utils - name: opensearch-observability repository: https://github.com/opensearch-project/observability.git - ref: '1.3' + ref: 05d4d25962909fa617d7bf2f9b41dcba107a3b19 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -107,7 +107,7 @@ components: - common-utils - name: anomaly-detection repository: https://github.com/opensearch-project/anomaly-detection.git - ref: '1.3' + ref: 1bc68718c8db0584dc984978e0dc11e34cef7b31 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -119,7 +119,7 @@ components: - job-scheduler - name: asynchronous-search repository: https://github.com/opensearch-project/asynchronous-search.git - ref: '1.3' + ref: 214092a6b12e3880a825aae30373461e71390697 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -130,7 +130,7 @@ components: - common-utils - name: k-NN repository: https://github.com/opensearch-project/k-NN.git - ref: '1.3' + ref: ff38dc491366bb9193b5fd947e9a7e0cc9556721 checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -139,7 +139,7 @@ components: - windows - name: sql repository: https://github.com/opensearch-project/sql.git - ref: '1.3' + ref: 924fb301420385a08e38de2f80b63d60bfc6d792 checks: - gradle:properties:version - gradle:dependencies:opensearch.version: plugin @@ -150,7 +150,7 @@ components: - ml-commons - name: index-management repository: https://github.com/opensearch-project/index-management.git - ref: '1.3' + ref: d658843be9855446bbd82b3ee9b21d4c4285ce31 checks: - gradle:properties:version - gradle:dependencies:opensearch.version diff --git a/manifests/1.3.18/opensearch-dashboards-1.3.18.yml b/manifests/1.3.18/opensearch-dashboards-1.3.18.yml index fbb2ad2bd8..44b15f13fe 100644 --- a/manifests/1.3.18/opensearch-dashboards-1.3.18.yml +++ b/manifests/1.3.18/opensearch-dashboards-1.3.18.yml @@ -9,31 +9,31 @@ ci: components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git - ref: '1.3' + ref: 26d95c2bb5d43e3d540d4612884ff98f66938046 - name: functionalTestDashboards repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git - ref: '1.3' + ref: 8b999677608cbe09f3f412ba1b3b57c9731a7ffc - name: anomalyDetectionDashboards repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin - ref: '1.3' + ref: 1da96108484cf891f071ca54634b8df815dccec9 - name: ganttChartDashboards repository: https://github.com/opensearch-project/dashboards-visualizations.git - ref: '1.3' + ref: 968bcf9f028e7fdd503eaf25e0c877efff0c9ed8 - name: observabilityDashboards repository: https://github.com/opensearch-project/dashboards-observability.git - ref: '1.3' + ref: a1d4df6bc56b04197f6fc3611c8e7aee6fd51b9f - name: alertingDashboards repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git - ref: '1.3' + ref: 7c133586fec9cc4d0f3e4d98b2182e2e8563f3dc - name: indexManagementDashboards repository: https://github.com/opensearch-project/index-management-dashboards-plugin - ref: '1.3' + ref: 189fd6b9d81059d4d0485e3a84e2aecebf020b58 - name: reportsDashboards repository: https://github.com/opensearch-project/dashboards-reporting.git - ref: '1.3' + ref: 45fa281c2f2f727b460d2d8cfdbb61d502046d73 - name: securityDashboards repository: https://github.com/opensearch-project/security-dashboards-plugin.git - ref: '1.3' + ref: 7d471b1db91f4452e1fb03ec31cea2862ff69017 - name: queryWorkbenchDashboards repository: https://github.com/opensearch-project/dashboards-query-workbench.git - ref: '1.3' + ref: 38cebc903e5a3a3247b6493002f235d6389e535d diff --git a/src/test_workflow/benchmark_test/benchmark_test_suite.py b/src/test_workflow/benchmark_test/benchmark_test_suite.py index c8dd834d69..1edf598b6b 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_suite.py +++ b/src/test_workflow/benchmark_test/benchmark_test_suite.py @@ -74,25 +74,31 @@ def __init__( self.command += f" --telemetry-params '{self.args.telemetry_params}'" if self.security: - self.command += f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\',basic_auth_password:\'{self.password}\'"' + self.command += (f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\',' + f'basic_auth_password:\'{self.password}\'" --results-file=final_result.md') else: - self.command += ' --client-options="timeout:300"' + self.command += ' --client-options="timeout:300" --results-file=final_result.md' def execute(self) -> None: log_info = f"Executing {self.command.replace(self.endpoint, len(self.endpoint) * '*').replace(self.args.username, len(self.args.username) * '*')}" logging.info(log_info.replace(self.password, len(self.password) * '*') if self.password else log_info) try: subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True) - if self.args.cluster_endpoint: + if self.args.cluster_endpoint or self.args.distribution_url: self.convert() finally: self.cleanup() def convert(self) -> None: with TemporaryDirectory() as work_dir: - subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark/test_executions/. {str(work_dir.path)}", cwd=os.getcwd(), shell=True) + subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark" + f"/test_executions/. {str(work_dir.path)}", cwd=os.getcwd(), shell=True) + subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark" + f"/final_result.md {str(work_dir.path)}", cwd=os.getcwd(), shell=True) file_path = glob.glob(os.path.join(str(work_dir.path), "*", "test_execution.json")) - shutil.copy(file_path[0], os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.json")) + final_results_file = glob.glob(os.path.join(str(work_dir.path), "final_result.md")) + shutil.copy(file_path[0], os.path.join('/tmp', f"test_execution_{self.args.stack_suffix}.json")) + shutil.copy(final_results_file[0], os.path.join('/tmp', f"final_result_{self.args.stack_suffix}.md")) with open(file_path[0]) as file: data = json.load(file) formatted_data = pd.json_normalize(data["results"]["op_metrics"]) diff --git a/tests/jenkins/TestBenchmarkPullRequest.groovy b/tests/jenkins/TestBenchmarkPullRequest.groovy new file mode 100644 index 0000000000..39fbf44aa4 --- /dev/null +++ b/tests/jenkins/TestBenchmarkPullRequest.groovy @@ -0,0 +1,198 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource + +class TestBenchmarkPullRequest extends BuildPipelineTest { + @Override + @Before + void setUp() { + super.setUp() + + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('6.6.0') + .allowOverride(true) + .implicit(true) + .targetPath('vars') + .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) + .build() + ) + + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod('unstash', [String.class], null) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + helper.registerAllowedMethod("downloadBuildManifest", [Map], { + c -> lib.jenkins.BuildManifest.new(readYaml(file: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml')) + }) + helper.registerAllowedMethod('parameterizedCron', [String], null) + helper.registerAllowedMethod("cfnDescribe", [Map]) + helper.registerAllowedMethod("cfnDelete", [Map]) + + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('env', ['BUILD_NUMBER': '307']) + binding.setVariable('BUILD_NUMBER', '307') + binding.setVariable('BUILD_URL', 'test://artifact.url') + binding.setVariable('BUILD_ID', '1234') + binding.setVariable('BUNDLE_MANIFEST', '') + binding.setVariable('BUNDLE_MANIFEST_URL', '') + binding.setVariable('DISTRIBUTION_URL', 'https://artifacts.com/artifact.tar.gz') + binding.setVariable('DISTRIBUTION_VERSION', '3.0.0') + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') + binding.setVariable('GITHUB_USER', 'test_user') + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('HAS_SECURITY', 'false') + binding.setVariable('SINGLE_NODE_CLUSTER', 'true') + binding.setVariable('MIN_DISTRIBUTION', 'true') + binding.setVariable('USE_50_PERCENT_HEAP', 'true') + binding.setVariable('SUFFIX', '1234') + binding.setVariable('MANAGER_NODE_COUNT', '') + binding.setVariable('DATA_NODE_COUNT', '') + binding.setVariable('ENABLE_REMOTE_STORE', 'false') + binding.setVariable('USER_TAGS', 'run-type:test') + binding.setVariable('WORKLOAD_PARAMS', '') + binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts') + binding.setVariable('EXCLUDE_TASKS', '') + binding.setVariable('INCLUDE_TASKS', '') + binding.setVariable('ADDITIONAL_CONFIG', '') + binding.setVariable('CLIENT_NODE_COUNT', '') + binding.setVariable('INGEST_NODE_COUNT', '') + binding.setVariable('ML_NODE_COUNT', '') + binding.setVariable('DATA_NODE_STORAGE', '100') + binding.setVariable('ML_NODE_STORAGE', '') + binding.setVariable('DATA_INSTANCE_TYPE', 'r5-4xlarge') + binding.setVariable('JVM_SYS_PROPS', '') + binding.setVariable('CAPTURE_NODE_STAT', 'false') + binding.setVariable('CAPTURE_SEGMENT_REPLICATION_STAT', 'false') + binding.setVariable('JOB_NAME', 'benchmark-test') + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') + binding.setVariable('STAGE_NAME', 'test_stage') + binding.setVariable('TEST_WORKLOAD', 'nyc-taxis') + binding.setVariable('WEBHOOK_URL', 'test://artifact.url') + binding.setVariable('TELEMETRY_PARAMS', '') + binding.setVariable('pull_request_number', '1234') + binding.setVariable('pull_request', 1234) + binding.setVariable('repository','opensearch-project/OpenSearch') + + helper.registerAllowedMethod("GenericTrigger", [Map], { println 'GenericTrigger called with params: ' + it }) + } + + @Test + public void testBenchmarkPullRequestGenericCause_verifyPipeline() { + binding.getVariable('currentBuild').rawBuild = [:] + binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" } + + super.testPipeline("jenkins/opensearch/benchmark-pull-request.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile") + } + + @Test + public void testBenchmarkPullRequestUserIdCause_verifyPipeline() { + binding.getVariable('currentBuild').rawBuild = [:] + binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.UserIdCause@123abc" } + + super.testPipeline("jenkins/opensearch/benchmark-pull-request.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile") + } + + @Test + void testBenchmarkPullRequest_verifyScriptExecutions() { + binding.getVariable('currentBuild').rawBuild = [:] + binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" } + + runScript("jenkins/opensearch/benchmark-pull-request.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems( + "set +x && ./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --single-node --min-distribution --use-50-percent-heap --suffix 307 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100".toString() + )) + + def testGhCliCommand = getCommandExecutions('sh', 'gh').findAll { + shCommand -> shCommand.contains('gh') + } + assertThat(testGhCliCommand.size(), equalTo(1)) + assertThat(testGhCliCommand, hasItem('gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md')) + } + + @Test + void testBenchmarkPullRequest_verifyJob_failure(){ + + binding.getVariable('currentBuild').rawBuild = [:] + binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" } + helper.registerAllowedMethod('sh', [String.class], { String cmd -> + updateBuildStatus('FAILURE') + }) + def result = runScript("jenkins/opensearch/benchmark-pull-request.jenkinsfile") + + assertJobStatusFailure() + assertCallStack() + assertCallStack().contains("gh pr comment 1234 --repo opensearch-project/OpenSearch --body The benchmark job test://artifact.url failed.") + } + + @Test + void testBenchmarkPullRequest_verifyJob_aborted() throws Exception { + + binding.getVariable('currentBuild').rawBuild = [:] + binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" } + helper.registerAllowedMethod("cfnDescribe", [Map.class]) { args -> return true} + helper.registerAllowedMethod('sh', [String.class], { String cmd -> + updateBuildStatus('ABORTED') + }) + runScript("jenkins/opensearch/benchmark-pull-request.jenkinsfile") + + assertJobStatusAborted() + assertCallStack() + assertCallStack().contains("cfnDescribe({stack=opensearch-infra-stack-307})") + assertCallStack().contains("cfnDelete({stack=opensearch-infra-stack-307, pollInterval=1000})") + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt new file mode 100644 index 0000000000..d4c8a098de --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt @@ -0,0 +1,41 @@ + benchmark-pull-request.run() + benchmark-pull-request.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-pull-request.library({identifier=jenkins@6.4.3, retriever=null}) + benchmark-pull-request.pipeline(groovy.lang.Closure) + benchmark-pull-request.timeout({time=24, unit=HOURS}) + benchmark-pull-request.logRotator({daysToKeepStr=30}) + benchmark-pull-request.buildDiscarder(null) + benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) + benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) + benchmark-pull-request.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-pull-request.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.add(benchmark-test, Benchmark tests for https://artifacts.com/artifact.tar.gz) + benchmark-pull-request.writeFile({file=messages/benchmark-test.msg, text=Benchmark tests for https://artifacts.com/artifact.tar.gz}) + benchmark-pull-request.stash({includes=messages/*, name=messages-benchmark-test}) + benchmark-pull-request.stage(benchmark-pull-request, groovy.lang.Closure) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.echo(security-enabled: false) + benchmark-pull-request.runBenchmarkTestScript({distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=true, minDistribution=true, use50PercentHeap=true, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, userTag=run-type:test,security-enabled:false, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.6.0, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --single-node --min-distribution --use-50-percent-heap --suffix 307 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100) + benchmark-pull-request.sh(cp /tmp/final_result_307.md final_result_307.md) + benchmark-pull-request.sh(sed -i '1i\#### Benchmark Results for Job: test://artifact.url' final_result_307.md) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + benchmark-pull-request.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + benchmark-pull-request.sh(gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md) + benchmark-pull-request.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt new file mode 100644 index 0000000000..500dd1fc90 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt @@ -0,0 +1,38 @@ + benchmark-pull-request.run() + benchmark-pull-request.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-pull-request.library({identifier=jenkins@6.4.3, retriever=null}) + benchmark-pull-request.pipeline(groovy.lang.Closure) + benchmark-pull-request.timeout({time=24, unit=HOURS}) + benchmark-pull-request.logRotator({daysToKeepStr=30}) + benchmark-pull-request.buildDiscarder(null) + benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) + benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) + benchmark-pull-request.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-pull-request.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.add(benchmark-test, Benchmark tests for https://artifacts.com/artifact.tar.gz) + benchmark-pull-request.writeFile({file=messages/benchmark-test.msg, text=Benchmark tests for https://artifacts.com/artifact.tar.gz}) + benchmark-pull-request.stash({includes=messages/*, name=messages-benchmark-test}) + benchmark-pull-request.stage(benchmark-pull-request, groovy.lang.Closure) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.echo(security-enabled: false) + benchmark-pull-request.runBenchmarkTestScript({distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=true, minDistribution=true, use50PercentHeap=true, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, userTag=run-type:test,security-enabled:false, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.6.0, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --single-node --min-distribution --use-50-percent-heap --suffix 307 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100) + benchmark-pull-request.sh(cp /tmp/final_result_307.md final_result_307.md) + benchmark-pull-request.sh(sed -i '1i\#### Benchmark Results for Job: test://artifact.url' final_result_307.md) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request.jenkinsfile.txt new file mode 100644 index 0000000000..0c07c6edba --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request.jenkinsfile.txt @@ -0,0 +1,50 @@ + benchmark-pull-request.run() + benchmark-pull-request.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-pull-request.library({identifier=jenkins@6.6.0, retriever=null}) + benchmark-pull-request.pipeline(groovy.lang.Closure) + benchmark-pull-request.timeout({time=24, unit=HOURS}) + benchmark-pull-request.logRotator({daysToKeepStr=30}) + benchmark-pull-request.buildDiscarder(null) + benchmark-pull-request.echo(Executing on agent [label:none]) + benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) + benchmark-pull-request.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test]) + benchmark-pull-request.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.add(benchmark-test, Benchmark tests for https://artifacts.com/artifact.tar.gz) + benchmark-pull-request.writeFile({file=messages/benchmark-test.msg, text=Benchmark tests for https://artifacts.com/artifact.tar.gz}) + benchmark-pull-request.stash({includes=messages/*, name=messages-benchmark-test}) + benchmark-pull-request.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + benchmark-pull-request.stage(benchmark-pull-request, groovy.lang.Closure) + benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test]) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.echo(security-enabled: false) + benchmark-pull-request.runBenchmarkTestScript({distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, userTag=run-type:test,security-enabled:false, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.6.0, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100) + benchmark-pull-request.stash({includes=test_execution*.csv, name=benchmark}) + benchmark-pull-request.stash({includes=test_execution*.json, name=benchmark-json}) + benchmark-pull-request.node(Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test, groovy.lang.Closure) + benchmark-pull-request.unstash(benchmark) + benchmark-pull-request.unstash(benchmark-json) + benchmark-pull-request.archiveArtifacts({artifacts=test_execution*.csv}) + benchmark-pull-request.sh(sed -i '1i\#### Benchmark Results for Job: test://artifact.url' final_result_307.md) + benchmark-pull-request.script(groovy.lang.Closure) + benchmark-pull-request.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + benchmark-pull-request.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + benchmark-pull-request.sh(gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md) + benchmark-pull-request.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt index 4c2b164746..7e0a4eaba5 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt @@ -45,6 +45,8 @@ H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.11.1;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-211","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-211","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.7.0/opensearch-2.7.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.7.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-270","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-270","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.12.0/opensearch-2.12.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.12.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.13.0/opensearch-2.13.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.13.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.14.0/opensearch-2.14.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.14.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt index 4f08091dff..fbc07e8bca 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt @@ -45,6 +45,8 @@ H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.11.1;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-211","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-211","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.7.0/opensearch-2.7.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.7.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-270","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-270","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.12.0/opensearch-2.12.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.12.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.13.0/opensearch-2.13.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.13.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.14.0/opensearch-2.14.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.14.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt index 726f1cbb3b..4ad81a2b07 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt @@ -45,6 +45,8 @@ H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.16.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"};EXCLUDE_TASKS=type:search + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.11.1;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-211","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-211","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.7.0/opensearch-2.7.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.7.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-270","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-270","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.12.0/opensearch-2.12.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.12.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.13.0/opensearch-2.13.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.13.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.14.0/opensearch-2.14.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.14.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py index 6606b685a3..5cda3e75a1 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py @@ -48,7 +48,7 @@ def test_execute_default(self, mock_check_call: Mock) -> None: self.assertEqual(mock_check_call.call_count, 2) self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:1.6.0 execute-test ' - f'--workload=nyc_taxis --pipeline=benchmark-only --target-hosts=abc.com:80 --client-options="timeout:300"') + f'--workload=nyc_taxis --pipeline=benchmark-only --target-hosts=abc.com:80 --client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -64,7 +64,7 @@ def test_execute_security_enabled_version_212_or_greater(self, mock_convert: Moc f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:1.6.0 execute-test' f' --workload=nyc_taxis --pipeline=benchmark-only ' f'--target-hosts=abc.com:443 ' - f'--client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'"') + f'--client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -80,7 +80,7 @@ def test_execute_security_enabled(self, mock_convert: Mock, mock_check_call: Moc f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:1.6.0 execute-test ' '--workload=nyc_taxis --pipeline=benchmark-only ' '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' - 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -104,7 +104,7 @@ def test_execute_default_with_optional_args(self, mock_convert: Mock, mock_check '--pipeline=benchmark-only --target-hosts=abc.com:80 ' '--workload-params \'{"number_of_replicas":"1"}\' ' '--user-tag="key1:value1,key2:value2" --telemetry node-stats, --telemetry-params \'{"example_key":"example_value"}\' ' - '--client-options="timeout:300"') + '--client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -125,7 +125,7 @@ def test_execute_default_with_no_telemetry_params(self, mock_convert: Mock, mock '--pipeline=benchmark-only --target-hosts=abc.com:80 ' '--workload-params \'{"number_of_replicas":"1"}\' ' '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' - '--client-options="timeout:300"') + '--client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -148,7 +148,7 @@ def test_execute_with_test_procedure_params(self, mock_convert: Mock, mock_check '--workload-params \'{"number_of_replicas":"1"}\' ' '--test-procedure="test-proc1,test-proc2" ' '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + '--client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -173,7 +173,7 @@ def test_execute_with_include_exclude_params(self, mock_convert: Mock, mock_chec '--exclude-tasks="task2,type:search" ' '--include-tasks="task1,type:index" ' '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + '--client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock) -> None: @@ -197,7 +197,7 @@ def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock) -> '--exclude-tasks="task2,type:search" ' '--include-tasks="task1,type:index" ' '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + '--client-options="timeout:300" --results-file=final_result.md') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -215,7 +215,7 @@ def test_execute_cluster_endpoint(self, mock_convert: Mock, mock_check_call: Moc f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:1.6.0 execute-test ' '--workload=nyc_taxis --pipeline=benchmark-only ' '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' - 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'" --results-file=final_result.md') @patch('pandas.json_normalize') @patch('pandas.read_csv') @@ -239,11 +239,12 @@ def test_convert(self, mock_check_call: Mock, mock_copy: Mock, mock_get_terminal mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] + mock_glob.return_value = ['/mock/test_execution.json', '/mock/final_result.md'] test_suite.convert() mock_temp_directory.assert_called_once() - mock_copy.assert_called_once() - mock_check_call.assert_called_with(f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/test_executions/. /mock/temp/dir", cwd=os.getcwd(), shell=True) + mock_check_call.assert_any_call(f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/test_executions/. /mock/temp/dir", cwd=os.getcwd(), shell=True) + mock_check_call.assert_any_call(f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/final_result.md /mock/temp/dir", cwd=os.getcwd(), shell=True) + mock_open.assert_called_once_with("/mock/test_execution.json") mock_json_load.assert_called_once() mock_json_normalize.assert_called_once()