diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile new file mode 100644 index 0000000000..ed67151de0 --- /dev/null +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([ + + $class: 'GitSCMSource', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', +])) + +pipeline { + agent none + options { + timeout(time: 24, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) + } + environment { + AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' + } +parameters { + string( + name: 'CLUSTER_ENDPOINT', + description: 'Provide an endpoint to a cluster for running benchmark tests against it.', + trim: true + ) + booleanParam( + name: 'SECURITY_ENABLED', + description: 'Mention if the cluster is secured or insecured.', + defaultValue: false, + ) + string( + + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads.', + defaultValue: 'nyc_taxis', + trim: true + ) + string( + name: 'USER_TAGS', + description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ', + trim: true + ) + string( + name: 'WORKLOAD_PARAMS', + description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`', + trim: true + ) + string( + name: 'TEST_PROCEDURE', + description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`', + trim: true + ) + string( + name: 'EXCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + string( + name: 'INCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + booleanParam( + name: 'CAPTURE_NODE_STAT', + description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.', + defaultValue: false + ) + booleanParam( + name: 'CAPTURE_SEGMENT_REPLICATION_STAT', + description: 'Enable opensearch-benchmark segment-replication-stats telemetry to capture metrics such as replication lag.', + defaultValue: false + ) + string( + name: 'TELEMETRY_PARAMS', + description: 'Allows to set parameters for telemetry devices. Use json type. e.g.,{"node-stats-include-indices":"true","node-stats-include-indices-metrics":"segments"}', + trim: true + ) + } + + stages { + stage('validate-and-set-parameters') { + agent { label AGENT_LABEL } + steps { + script { + if (CLUSTER_ENDPOINT == '') { + currentBuild.result = 'ABORTED' + error("Benchmark Tests failed to start. Provide CLUSTER_ENDPOINT to run tests") + } + } + } + post { + always { + postCleanup() + } + } + } + stage('benchmark-test-with-cluster') { + agent { label AGENT_LABEL } + steps { + script { + echo "security-enabled: ${SECURITY_ENABLED}" + + runBenchmarkTestScript( + endpoint: CLUSTER_ENDPOINT, + insecure: !(params.SECURITY_ENABLED), + workload: TEST_WORKLOAD, + userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}", + workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, + telemetryParams: TELEMETRY_PARAMS + ) + } + post { + always { + postCleanup() + } + } + } + } + } +} diff --git a/jenkins/opensearch/benchmark-test.jenkinsfile b/jenkins/opensearch/benchmark-test.jenkinsfile index 548eee6706..c7a621cddb 100644 --- a/jenkins/opensearch/benchmark-test.jenkinsfile +++ b/jenkins/opensearch/benchmark-test.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy new file mode 100644 index 0000000000..34535ad41c --- /dev/null +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource + +class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ + @Override + @Before + void setUp() { + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('6.4.1') + .allowOverride(true) + .implicit(true) + .targetPath('vars') + .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) + .build() + ) + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') + binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com') + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') + binding.setVariable('GITHUB_USER', 'test_user') + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('USER_TAGS', 'run-type:test') + binding.setVariable('WORKLOAD_PARAMS', '') + binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts') + binding.setVariable('EXCLUDE_TASKS', '') + binding.setVariable('INCLUDE_TASKS', '') + binding.setVariable('ADDITIONAL_CONFIG', '') + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('STAGE_NAME', 'test_stage') + binding.setVariable('TEST_WORKLOAD', 'nyc-taxis') + binding.setVariable('TELEMETRY_PARAMS', '{"telemetry_setting":"value"}') + + super.setUp() + } + + @Test + public void testRunSecureBenchmarkTestScript_verifyPipeline() { + addParam('SECURITY_ENABLED', true) + super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile") + } + + @Test + void testRunSecureBenchmarkTestScript_verifyArtifactDownloads() { + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(1)) + assertThat(s3DownloadCommands, hasItems( + "{file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}".toString() + )) + } + + + @Test + void testRunSecureBenchmarkTestScript_verifyScriptExecutions() { + addParam('SECURITY_ENABLED', true) + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems( + "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + )) + } + @Test + public void testRunSecureBenchmarkTestWithoutSecurity_verifyPipeline() { + addParam('SECURITY_ENABLED', true) + super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile") + } + + @Test + void testRunSecureBenchmarkTestScript_verifyWithoutSecurity() { + addParam('SECURITY_ENABLED', false) + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems( + "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} diff --git a/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy b/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy index 1081bdbf35..c857cda436 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy @@ -26,7 +26,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest { void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -131,7 +131,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItems( - "./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100 ".toString() + "./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100".toString() )) } diff --git a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy index 38be1b7d39..8d63f62e9a 100644 --- a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -140,7 +140,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100".toString() )) } diff --git a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy index 417668b176..388ee400aa 100644 --- a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -141,8 +141,8 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItems( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(), - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(), + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt index 0fbb585d46..04ad61de18 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -144,7 +144,7 @@ benchmark-test.echo(DISTRIBUTION_VERSION: 3.0.0) benchmark-test.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) @@ -156,7 +156,7 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt new file mode 100644 index 0000000000..8a4641830a --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt @@ -0,0 +1,33 @@ + benchmark-test-endpoint.run() + benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-test-endpoint.library({identifier=jenkins@6.4.1, retriever=null}) + benchmark-test-endpoint.pipeline(groovy.lang.Closure) + benchmark-test-endpoint.timeout({time=24, unit=HOURS}) + benchmark-test-endpoint.logRotator({daysToKeepStr=30}) + benchmark-test-endpoint.buildDiscarder(null) + benchmark-test-endpoint.echo(Executing on agent [label:none]) + benchmark-test-endpoint.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.echo(security-enabled: true) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt new file mode 100644 index 0000000000..8a4641830a --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt @@ -0,0 +1,33 @@ + benchmark-test-endpoint.run() + benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-test-endpoint.library({identifier=jenkins@6.4.1, retriever=null}) + benchmark-test-endpoint.pipeline(groovy.lang.Closure) + benchmark-test-endpoint.timeout({time=24, unit=HOURS}) + benchmark-test-endpoint.logRotator({daysToKeepStr=30}) + benchmark-test-endpoint.buildDiscarder(null) + benchmark-test-endpoint.echo(Executing on agent [label:none]) + benchmark-test-endpoint.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.echo(security-enabled: true) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt index 039cff6262..791650ba71 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -131,7 +131,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() @@ -150,7 +150,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() @@ -159,7 +159,7 @@ benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=type:search,scroll, includeTasks=type:search,scroll, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -176,7 +176,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt index ff240b560b..2ec4795f7a 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -131,7 +131,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -149,7 +149,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -158,7 +158,7 @@ benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:true,jenkins-build-id:307, workloadParams=, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -175,7 +175,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) @@ -195,7 +195,7 @@ Benchmark tests with security for 1234 completed}) benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -204,7 +204,7 @@ Benchmark tests with security for 1234 completed}) benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -221,7 +221,7 @@ Benchmark tests with security for 1234 completed}) BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test])