From ad18e8c3df44c69885e890bbc0596533d5b3cf26 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 13 Dec 2019 16:31:02 -0500 Subject: [PATCH 01/27] WIP Jenkinsfile for ES snapshot building --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 119 +++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .ci/es-snapshots/Jenkinsfile_build_es.groovy diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy new file mode 100644 index 00000000000000..4eb2e1873b085f --- /dev/null +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -0,0 +1,119 @@ +#!/bin/groovy + +library 'kibana-pipeline-library' +// kibanaLibrary.load() + +def BRANCH = params.BRANCH ?: 'master' // TODO + +if (!BRANCH) { + error "Parameter 'BRANCH' must be specified." +} + +timeout(time: 120, unit: 'MINUTES') { + timestamps { + ansiColor('xterm') { + node('linux && immutable') { + def scmVars = checkoutEs('master') + print scmVars + def GIT_COMMIT = scmVars.GIT_COMMIT + def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() + + buildArchives('to-archive') + + dir('to-archive') { + def uuid = UUID.randomUUID().toString() + def now = new Date() + def date = now.format("yyyyMMdd") + + def version + def destination + + def filesRaw = sh(script: "ls -1", returnStdout: true).trim() + def files = filesRaw.split("\n").collect { filename -> + // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + def parts = filename.replace("elasticsearch-oss", "oss").split("-") + + version = version ?: parts[1] + destination = destination ?: "${version}/archives/${date}_${GIT_COMMIT_SHORT}_${uuid}" + + return [ + filename: filename, + checksum: filename + '.sha512', + url: "https://storage.cloud.google.com/kibana-ci-es-snapshots/${destination}/${filename}".toString(), + version: parts[1], + platform: parts.size() >= 5 ? parts[3] : '', + architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', + license: parts[0] == 'oss' ? 'oss' : 'default', + ] + } + + sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' + + def manifest = [ + id: uuid, + bucket: "kibana-ci-es-snapshots/${destination}".toString(), + branch: BRANCH, + sha: GIT_COMMIT, + sha_short: GIT_COMMIT_SHORT, + version: version, + generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")), + archives: files, + ] + def manifestJson = toJSON(manifest).toString() + writeFile file: 'manifest.json', text: manifestJson + + upload(destination, '*.*') + + sh "cp manifest.json manifest-latest.json" + upload(version, 'manifest-latest.json') + } + } + } + } +} + +def checkoutEs(branch) { + return checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [ + $class: 'CloneOption', + noTags: false, + reference: '/var/lib/jenkins/.git-references/elasticsearch.git', + shallow: true + ] + ], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + name: 'origin', + refspec: '+refs/heads/master:refs/remotes/origin/master', + url: 'git@github.com:elastic/elasticsearch', + ]], + ]) +} + +def upload(destination, pattern) { + return googleStorageUpload( + credentialsId: 'kibana-ci-gcs-plugin', + bucket: "gs://kibana-ci-es-snapshots/${destination}", + pattern: pattern, + sharedPublicly: false, + showInline: false, + ) +} + +def buildArchives(destination) { + withEnv([ + "PATH=/var/lib/jenkins/.java/openjdk13/bin:${env.PATH}", // Probably won't even need this if this gets wrapped in runbld + ]) { + sh """ + ./gradlew -p distribution/archives assemble --parallel + mkdir -p ${destination} + find distribution/archives -type f \\( -name 'elasticsearch-*-*.tar.gz' -o -name 'elasticsearch-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\; + """ + } +} From d2139c4744f3f5e099868a21916a29cdfe879197 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 13 Dec 2019 17:25:28 -0500 Subject: [PATCH 02/27] Remove guid and add short sha to es snapshot destination --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index 4eb2e1873b085f..e7716c32ba97f0 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -18,12 +18,13 @@ timeout(time: 120, unit: 'MINUTES') { def GIT_COMMIT = scmVars.GIT_COMMIT def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() + sh 'rm -rf to-archive || true' // TODO remove buildArchives('to-archive') dir('to-archive') { def uuid = UUID.randomUUID().toString() def now = new Date() - def date = now.format("yyyyMMdd") + def date = now.format("yyyyMMdd-HHmmss") def version def destination @@ -35,7 +36,7 @@ timeout(time: 120, unit: 'MINUTES') { def parts = filename.replace("elasticsearch-oss", "oss").split("-") version = version ?: parts[1] - destination = destination ?: "${version}/archives/${date}_${GIT_COMMIT_SHORT}_${uuid}" + destination = destination ?: "${version}/archives/${date}_${GIT_COMMIT_SHORT}" return [ filename: filename, From 4736d3ce0504f033327b3ffd2ba063e2e121d6c3 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 13 Dec 2019 21:29:56 -0500 Subject: [PATCH 03/27] Fix new snapshot URLs and WIP kbn-es support for new snapshots --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 5 +- packages/kbn-es/src/artifact.js | 3 +- packages/kbn-es/src/custom_snapshots.js | 69 +++++++++++++++++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index e7716c32ba97f0..044c6f31419a5e 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -22,7 +22,6 @@ timeout(time: 120, unit: 'MINUTES') { buildArchives('to-archive') dir('to-archive') { - def uuid = UUID.randomUUID().toString() def now = new Date() def date = now.format("yyyyMMdd-HHmmss") @@ -41,7 +40,7 @@ timeout(time: 120, unit: 'MINUTES') { return [ filename: filename, checksum: filename + '.sha512', - url: "https://storage.cloud.google.com/kibana-ci-es-snapshots/${destination}/${filename}".toString(), + url: "https://storage.googleapis.com/kibana-ci-es-snapshots/${destination}/${filename}".toString(), version: parts[1], platform: parts.size() >= 5 ? parts[3] : '', architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', @@ -52,7 +51,6 @@ timeout(time: 120, unit: 'MINUTES') { sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' def manifest = [ - id: uuid, bucket: "kibana-ci-es-snapshots/${destination}".toString(), branch: BRANCH, sha: GIT_COMMIT, @@ -75,6 +73,7 @@ timeout(time: 120, unit: 'MINUTES') { } def checkoutEs(branch) { + // TODO wrap in a new retryWithDelay(15, 8){} return checkout([ $class: 'GitSCM', branches: [[name: branch]], diff --git a/packages/kbn-es/src/artifact.js b/packages/kbn-es/src/artifact.js index 19d95e82fe4800..cb48c2d4a35b0a 100644 --- a/packages/kbn-es/src/artifact.js +++ b/packages/kbn-es/src/artifact.js @@ -95,8 +95,9 @@ exports.Artifact = class Artifact { static async getSnapshot(license, version, log) { const urlVersion = `${encodeURIComponent(version)}-SNAPSHOT`; - const customSnapshotArtifactSpec = resolveCustomSnapshotUrl(urlVersion, license); + const customSnapshotArtifactSpec = await resolveCustomSnapshotUrl(urlVersion, license); if (customSnapshotArtifactSpec) { + console.log(customSnapshotArtifactSpec); return new Artifact(customSnapshotArtifactSpec, log); } diff --git a/packages/kbn-es/src/custom_snapshots.js b/packages/kbn-es/src/custom_snapshots.js index 74de3c2c792fdc..b6de4e89064f4e 100644 --- a/packages/kbn-es/src/custom_snapshots.js +++ b/packages/kbn-es/src/custom_snapshots.js @@ -16,16 +16,24 @@ * specific language governing permissions and limitations * under the License. */ - +const AbortController = require('abort-controller'); +const fetch = require('node-fetch'); const { basename } = require('path'); +const { createCliError } = require('./errors'); + function isVersionFlag(a) { return a.startsWith('--version'); } +function shouldUseUnverifiedSnapshot() { + return !!process.env.KBN_ES_SNAPSHOT_USE_UNVERIFIED; +} + function getCustomSnapshotUrl() { // force use of manually created snapshots until ReindexPutMappings fix if (!process.env.KBN_ES_SNAPSHOT_URL && !process.argv.some(isVersionFlag)) { + return; return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}'; } @@ -34,10 +42,67 @@ function getCustomSnapshotUrl() { } } -function resolveCustomSnapshotUrl(urlVersion, license) { +// TODO all of this stuff will probably move somewhere else +async function fetchSnapshotManifest(url) { + const abc = new AbortController(); + const resp = await fetch(url, { signal: abc.signal }); + const json = await resp.text(); + + return { abc, resp, json }; +} + +async function getSnapshotManifest(urlVersion, license) { + const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); + const desiredLicense = license === 'oss' ? 'oss' : 'default'; + + let { abc, resp, json } = await fetchSnapshotManifest( + `https://storage.googleapis.com/kibana-ci-es-snapshots/${desiredVersion}/manifest-latest${ + shouldUseUnverifiedSnapshot() ? '' : '-verified' + }.json` + ); + if (!shouldUseUnverifiedSnapshot() && resp.status === 404) { + ({ abc, resp, json } = await fetchSnapshotManifest( + `https://storage.googleapis.com/kibana-ci-es-snapshots-permanent/${desiredVersion}/manifest.json` + )); + } + + if (resp.status === 404) { + abc.abort(); + throw createCliError(`Snapshots for ${desiredVersion} are not available`); + } + + if (!resp.ok) { + abc.abort(); + throw new Error(`Unable to read artifact info: ${resp.statusText}\n ${json}`); + } + + const manifest = JSON.parse(json); + + const platform = process.platform === 'win32' ? 'windows' : process.platform; + const archive = manifest.archives.find( + archive => + archive.version === desiredVersion && + archive.platform === platform && + archive.license === desiredLicense + ); + + return { + url: archive.url, + checksumUrl: archive.url + '.sha512', + checksumType: 'sha512', + filename: archive.filename, + }; +} + +async function resolveCustomSnapshotUrl(urlVersion, license) { const customSnapshotUrl = getCustomSnapshotUrl(); if (!customSnapshotUrl) { + const manifest = getSnapshotManifest(urlVersion, license); + if (manifest) { + return manifest; + } + return; } From d126f45be49de459fc31696724f283177841aea6 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 13 Dec 2019 21:38:55 -0500 Subject: [PATCH 04/27] WIP verify es Jenkinsfile --- .ci/es-snapshots/Jenkinsfile_verify_es.groovy | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .ci/es-snapshots/Jenkinsfile_verify_es.groovy diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy new file mode 100644 index 00000000000000..3b1d654892bc88 --- /dev/null +++ b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy @@ -0,0 +1,61 @@ +#!/bin/groovy + +library 'kibana-pipeline-library' +kibanaLibrary.load() + +def ES_OSS_SNAPSHOT = 'https://storage.googleapis.com/kibana-ci-es-snapshots/8.0.0/archives/20191214-021702_b9fbc8dc74/elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz' +def ES_XPACK_SNAPSHOT = 'https://storage.googleapis.com/kibana-ci-es-snapshots/8.0.0/archives/20191214-021702_b9fbc8dc74/elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz' + +stage("Kibana Pipeline") { + timeout(time: 120, unit: 'MINUTES') { + timestamps { + ansiColor('xterm') { + catchError { + parallel([ + oss: { + withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { + parallel([ + 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), + 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), + 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), + 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), + 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), + 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), + 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), + 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), + 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), + 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), + 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), + 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), + 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), + ]), + ]) + } + }, + xpack: { + withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { + parallel([ + 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), + 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), + 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), + 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), + 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), + 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), + 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), + 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), + 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), + 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), + 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), + ]), + ]) + } + } + ]) + } + // kibanaPipeline.sendMail() + } + } + } +} From 2fb1e15e6b62faa199722465262f962adb47c8b5 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 15:39:21 -0500 Subject: [PATCH 05/27] Add retryWithDelay, fix some ES build logic --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 48 ++++++++++---------- vars/kibanaPipeline.groovy | 9 +--- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index 044c6f31419a5e..122184e4fbf812 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -4,6 +4,7 @@ library 'kibana-pipeline-library' // kibanaLibrary.load() def BRANCH = params.BRANCH ?: 'master' // TODO +BRANCH = BRANCH != 'master' ? BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists if (!BRANCH) { error "Parameter 'BRANCH' must be specified." @@ -13,8 +14,7 @@ timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { node('linux && immutable') { - def scmVars = checkoutEs('master') - print scmVars + def scmVars = checkoutEs(BRANCH) def GIT_COMMIT = scmVars.GIT_COMMIT def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() @@ -73,27 +73,19 @@ timeout(time: 120, unit: 'MINUTES') { } def checkoutEs(branch) { - // TODO wrap in a new retryWithDelay(15, 8){} - return checkout([ - $class: 'GitSCM', - branches: [[name: branch]], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [ - $class: 'CloneOption', - noTags: false, - reference: '/var/lib/jenkins/.git-references/elasticsearch.git', - shallow: true - ] - ], - submoduleCfg: [], - userRemoteConfigs: [[ - credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', - name: 'origin', - refspec: '+refs/heads/master:refs/remotes/origin/master', - url: 'git@github.com:elastic/elasticsearch', - ]], - ]) + retryWithDelay(8, 15) { + return checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + url: 'git@github.com:elastic/elasticsearch', + ]], + ]) + } } def upload(destination, pattern) { @@ -107,8 +99,16 @@ def upload(destination, pattern) { } def buildArchives(destination) { + def props = readProperties file: '.ci/java-versions.properties' withEnv([ - "PATH=/var/lib/jenkins/.java/openjdk13/bin:${env.PATH}", // Probably won't even need this if this gets wrapped in runbld + "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", // Probably won't even need this if this gets wrapped in runbld + + // these jenkins env vars trigger some automation in ES that we don't want + "BUILD_NUMBER=", + "JENKINS_URL=", + "BUILD_URL=", + "JOB_NAME=", + "NODE_NAME=", ]) { sh """ ./gradlew -p distribution/archives assemble --parallel diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 18f214554b444d..c778dd799f6e59 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -137,13 +137,8 @@ def jobRunner(label, useRamDisk, closure) { def scmVars // Try to clone from Github up to 8 times, waiting 15 secs between attempts - retry(8) { - try { - scmVars = checkout scm - } catch (ex) { - sleep 15 - throw ex - } + retryWithDelay(8, 15) { + scmVars = checkout scm } withEnv([ From 47025ca9ba7441e29ebf600d8474d866e1eae632 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 15:47:03 -0500 Subject: [PATCH 06/27] Missed a new file --- vars/retryWithDelay.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vars/retryWithDelay.groovy diff --git a/vars/retryWithDelay.groovy b/vars/retryWithDelay.groovy new file mode 100644 index 00000000000000..70d6f86a63ab2d --- /dev/null +++ b/vars/retryWithDelay.groovy @@ -0,0 +1,16 @@ +def call(retryTimes, delaySecs, closure) { + retry(retryTimes) { + try { + closure() + } catch (ex) { + sleep delaySecs + throw ex + } + } +} + +def call(retryTimes, Closure closure) { + call(retryTimes, 15, closure) +} + +return this From 12fa300a1afe4fdcd95713d32422cbdda50cdcec Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 15:47:26 -0500 Subject: [PATCH 07/27] Load kibana library --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index 122184e4fbf812..e6f2196fa22cef 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -1,7 +1,7 @@ #!/bin/groovy library 'kibana-pipeline-library' -// kibanaLibrary.load() +kibanaLibrary.load() def BRANCH = params.BRANCH ?: 'master' // TODO BRANCH = BRANCH != 'master' ? BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists From 6de523b19823385f9f6d5aa9b95db79d841f42e3 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 17:24:15 -0500 Subject: [PATCH 08/27] WIP: ES snapshot verification working --- .ci/es-snapshots/Jenkinsfile_verify_es.groovy | 111 ++++++++++-------- vars/esSnapshots.groovy | 50 ++++++++ vars/kibanaPipeline.groovy | 3 +- 3 files changed, 113 insertions(+), 51 deletions(-) create mode 100644 vars/esSnapshots.groovy diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy index 3b1d654892bc88..83bcdfdc2da9e6 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy @@ -3,59 +3,70 @@ library 'kibana-pipeline-library' kibanaLibrary.load() -def ES_OSS_SNAPSHOT = 'https://storage.googleapis.com/kibana-ci-es-snapshots/8.0.0/archives/20191214-021702_b9fbc8dc74/elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz' -def ES_XPACK_SNAPSHOT = 'https://storage.googleapis.com/kibana-ci-es-snapshots/8.0.0/archives/20191214-021702_b9fbc8dc74/elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz' +// TODO finalize these +def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION ?: "8.0.0" +def SNAPSHOT_ID = params.SNAPSHOT_ID ?: "20191216-202229_c1d075a7dab" -stage("Kibana Pipeline") { - timeout(time: 120, unit: 'MINUTES') { - timestamps { - ansiColor('xterm') { - catchError { - parallel([ - oss: { - withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { - parallel([ - 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), - 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), - 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - ]), - ]) - } - }, - xpack: { - withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { - parallel([ - 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - ]), - ]) - } +def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" +def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" + +timeout(time: 120, unit: 'MINUTES') { + timestamps { + ansiColor('xterm') { + catchError { + parallel([ + oss: { + withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { + parallel([ + 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), // TODO we just need to run integration tests from intake? + 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), + 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), + 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), + 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), + 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), + 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), + 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), + 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), + 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), + 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), + 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), + 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), + ]), + ]) + } + }, + xpack: { + withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { + parallel([ + 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), // TODO we just need to run integration tests from intake? + 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), + 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), + 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), + 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), + 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), + 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), + 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), + 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), + 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), + 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), + ]), + ]) } - ]) - } - // kibanaPipeline.sendMail() + } + ]) + + promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) } + + sendKibanaMail() } } } + +def promoteSnapshot(snapshotVersion, snapshotId) { + node('linux && immutable') { + esSnapshots.promote(snapshotVersion, snapshotId) + } +} diff --git a/vars/esSnapshots.groovy b/vars/esSnapshots.groovy new file mode 100644 index 00000000000000..e3b2844e8ede9b --- /dev/null +++ b/vars/esSnapshots.groovy @@ -0,0 +1,50 @@ +def promote(snapshotVersion, snapshotId) { + def snapshotDestination = "${snapshotVersion}/archives/${snapshotId}" + def MANIFEST_URL = "https://storage.googleapis.com/kibana-ci-es-snapshots/${snapshotDestination}/manifest.json" + + dir('verified-manifest') { + def verifiedSnapshotFilename = 'manifest-latest-verified.json' + + sh """ + curl -O '${MANIFEST_URL}' + mv manifest.json ${verifiedSnapshotFilename} + """ + + googleStorageUpload( + credentialsId: 'kibana-ci-gcs-plugin', + bucket: "gs://kibana-ci-es-snapshots/${snapshotVersion}", + pattern: verifiedSnapshotFilename, + sharedPublicly: false, + showInline: false, + ) + } + + // This would probably be more efficient if we could just copy using gsutil and specifying buckets for src and dest + // But we don't currently have access to the GCS credentials in a way that can be consumed easily from here... + dir('transfer-to-permanent') { + googleStorageDownload( + credentialsId: 'kibana-ci-gcs-plugin', + bucketUri: "gs://kibana-ci-es-snapshots/${snapshotDestination}/*", + localDirectory: '.', + pathPrefix: snapshotDestination, + ) + + def manifestJson = readFile file: 'manifest.json' + writeFile( + file: 'manifest.json', + text: manifestJson.replace("kibana-ci-es-snapshots/${snapshotDestination}", "kibana-ci-es-snapshots-permanent/${snapshotVersion}") + ) + + // Ideally we would have some delete logic here before uploading, + // But we don't currently have access to the GCS credentials in a way that can be consumed easily from here... + googleStorageUpload( + credentialsId: 'kibana-ci-gcs-plugin', + bucket: "gs://kibana-ci-es-snapshots-permanent/${snapshotVersion}", + pattern: '*.*', + sharedPublicly: false, + showInline: false, + ) + } +} + +return this diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index c778dd799f6e59..c2da4acea715cd 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -237,7 +237,8 @@ def sendKibanaMail() { def buildStatus = buildUtils.getBuildStatus() if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED') { emailext( - to: 'build-kibana@elastic.co', + // to: 'build-kibana@elastic.co', + to: 'brian.seeders@elastic.co', // TODO change this back subject: "${env.JOB_NAME} - Build # ${env.BUILD_NUMBER} - ${buildStatus}", body: '${SCRIPT,template="groovy-html.template"}', mimeType: 'text/html', From 04abb3f4ed81f84c75fe8fc9cd197d8abd10b0f5 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 17:39:40 -0500 Subject: [PATCH 09/27] wip --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 113 ++++++++++-------- .ci/es-snapshots/Jenkinsfile_verify_es.groovy | 4 +- 2 files changed, 68 insertions(+), 49 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index e6f2196fa22cef..6b5f92b417d212 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -14,59 +14,76 @@ timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { node('linux && immutable') { - def scmVars = checkoutEs(BRANCH) - def GIT_COMMIT = scmVars.GIT_COMMIT - def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() - - sh 'rm -rf to-archive || true' // TODO remove - buildArchives('to-archive') - - dir('to-archive') { - def now = new Date() - def date = now.format("yyyyMMdd-HHmmss") - - def version - def destination - - def filesRaw = sh(script: "ls -1", returnStdout: true).trim() - def files = filesRaw.split("\n").collect { filename -> - // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - def parts = filename.replace("elasticsearch-oss", "oss").split("-") - - version = version ?: parts[1] - destination = destination ?: "${version}/archives/${date}_${GIT_COMMIT_SHORT}" - - return [ - filename: filename, - checksum: filename + '.sha512', - url: "https://storage.googleapis.com/kibana-ci-es-snapshots/${destination}/${filename}".toString(), - version: parts[1], - platform: parts.size() >= 5 ? parts[3] : '', - architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', - license: parts[0] == 'oss' ? 'oss' : 'default', + catchError { + def VERSION + def SNAPSHOT_ID + def DESTINATION + + def scmVars = checkoutEs(BRANCH) + def GIT_COMMIT = scmVars.GIT_COMMIT + def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() + + sh 'rm -rf to-archive || true' // TODO remove + buildArchives('to-archive') + + dir('to-archive') { + def now = new Date() + def date = now.format("yyyyMMdd-HHmmss") + + def filesRaw = sh(script: "ls -1", returnStdout: true).trim() + def files = filesRaw.split("\n").collect { filename -> + // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + def parts = filename.replace("elasticsearch-oss", "oss").split("-") + + VERSION = VERSION ?: parts[1] + SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" + DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" + + return [ + filename: filename, + checksum: filename + '.sha512', + url: "https://storage.googleapis.com/kibana-ci-es-snapshots/${DESTINATION}/${filename}".toString(), + version: parts[1], + platform: parts.size() >= 5 ? parts[3] : '', + architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', + license: parts[0] == 'oss' ? 'oss' : 'default', + ] + } + + sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' + + def manifest = [ + bucket: "kibana-ci-es-snapshots/${DESTINATION}".toString(), + branch: BRANCH, + sha: GIT_COMMIT, + sha_short: GIT_COMMIT_SHORT, + version: VERSION, + generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")), + archives: files, ] - } - - sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' + def manifestJson = toJSON(manifest).toString() + writeFile file: 'manifest.json', text: manifestJson - def manifest = [ - bucket: "kibana-ci-es-snapshots/${destination}".toString(), - branch: BRANCH, - sha: GIT_COMMIT, - sha_short: GIT_COMMIT_SHORT, - version: version, - generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")), - archives: files, - ] - def manifestJson = toJSON(manifest).toString() - writeFile file: 'manifest.json', text: manifestJson + upload(DESTINATION, '*.*') - upload(destination, '*.*') + sh "cp manifest.json manifest-latest.json" + upload(VERSION, 'manifest-latest.json') + } - sh "cp manifest.json manifest-latest.json" - upload(version, 'manifest-latest.json') + build( + propagate: false, + wait: false, + job: 'es-snapshot-verification', + parameters: [ + string(name: 'BRANCH', value: 'master'), // TODO + string(name: 'SNAPSHOT_VERSION', value: VERSION), + string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), + ] + ) } + + kibanaPipeline.sendMail() } } } diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy index 83bcdfdc2da9e6..2c57de7ed5dcf9 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy @@ -10,6 +10,8 @@ def SNAPSHOT_ID = params.SNAPSHOT_ID ?: "20191216-202229_c1d075a7dab" def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" +// TODO add ability to change scm based on BRANCH passed in + timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { @@ -60,7 +62,7 @@ timeout(time: 120, unit: 'MINUTES') { promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) } - sendKibanaMail() + kibanaPipeline.sendMail() } } } From 56fb41b1a1d7516cb5315ee40dcb3727d8925b94 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 16 Dec 2019 18:21:38 -0500 Subject: [PATCH 10/27] Fix branch_specifier --- .ci/es-snapshots/Jenkinsfile_build_es.groovy | 2 +- .ci/es-snapshots/Jenkinsfile_verify_es.groovy | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es.groovy index 6b5f92b417d212..0f9721bc440a0a 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es.groovy @@ -76,7 +76,7 @@ timeout(time: 120, unit: 'MINUTES') { wait: false, job: 'es-snapshot-verification', parameters: [ - string(name: 'BRANCH', value: 'master'), // TODO + string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO string(name: 'SNAPSHOT_VERSION', value: VERSION), string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), ] diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy index 2c57de7ed5dcf9..21a29cddc660aa 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_verify_es.groovy @@ -10,8 +10,6 @@ def SNAPSHOT_ID = params.SNAPSHOT_ID ?: "20191216-202229_c1d075a7dab" def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" -// TODO add ability to change scm based on BRANCH passed in - timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { From 36939b0e9e8e5c6c952893e04988a6124e1584e2 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 18 Dec 2019 11:35:40 -0500 Subject: [PATCH 11/27] Create job to trigger es build jobs --- ...e_build_es.groovy => Jenkinsfile_build_es} | 14 +++++++------- .ci/es-snapshots/Jenkinsfile_trigger_build_es | 19 +++++++++++++++++++ ...verify_es.groovy => Jenkinsfile_verify_es} | 0 3 files changed, 26 insertions(+), 7 deletions(-) rename .ci/es-snapshots/{Jenkinsfile_build_es.groovy => Jenkinsfile_build_es} (90%) create mode 100644 .ci/es-snapshots/Jenkinsfile_trigger_build_es rename .ci/es-snapshots/{Jenkinsfile_verify_es.groovy => Jenkinsfile_verify_es} (100%) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es.groovy b/.ci/es-snapshots/Jenkinsfile_build_es similarity index 90% rename from .ci/es-snapshots/Jenkinsfile_build_es.groovy rename to .ci/es-snapshots/Jenkinsfile_build_es index 0f9721bc440a0a..2445ddc56e990e 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es.groovy +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -3,11 +3,11 @@ library 'kibana-pipeline-library' kibanaLibrary.load() -def BRANCH = params.BRANCH ?: 'master' // TODO -BRANCH = BRANCH != 'master' ? BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists +def ES_BRANCH = params.ES_BRANCH ?: 'master' // TODO +ES_BRANCH = ES_BRANCH != 'master' ? ES_BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists -if (!BRANCH) { - error "Parameter 'BRANCH' must be specified." +if (!ES_BRANCH) { + error "Parameter 'ES_BRANCH' must be specified." } timeout(time: 120, unit: 'MINUTES') { @@ -19,7 +19,7 @@ timeout(time: 120, unit: 'MINUTES') { def SNAPSHOT_ID def DESTINATION - def scmVars = checkoutEs(BRANCH) + def scmVars = checkoutEs(ES_BRANCH) def GIT_COMMIT = scmVars.GIT_COMMIT def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() @@ -55,7 +55,7 @@ timeout(time: 120, unit: 'MINUTES') { def manifest = [ bucket: "kibana-ci-es-snapshots/${DESTINATION}".toString(), - branch: BRANCH, + branch: ES_BRANCH, sha: GIT_COMMIT, sha_short: GIT_COMMIT_SHORT, version: VERSION, @@ -76,7 +76,7 @@ timeout(time: 120, unit: 'MINUTES') { wait: false, job: 'es-snapshot-verification', parameters: [ - string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO + string(name: 'branch_specifier', value: branch_specifier), string(name: 'SNAPSHOT_VERSION', value: VERSION), string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), ] diff --git a/.ci/es-snapshots/Jenkinsfile_trigger_build_es b/.ci/es-snapshots/Jenkinsfile_trigger_build_es new file mode 100644 index 00000000000000..a765fde511ff24 --- /dev/null +++ b/.ci/es-snapshots/Jenkinsfile_trigger_build_es @@ -0,0 +1,19 @@ +#!/bin/groovy + +// TODO is there a good place to grab a list of these? +def branches = [ + 'master', + // '7.4', +] + +branches.each { branch -> + build( + propagate: false, + wait: false, + job: 'build-es', + parameters: [ + string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO + string(name: 'ES_BRANCH', value: branch), + ] + ) +} diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es.groovy b/.ci/es-snapshots/Jenkinsfile_verify_es similarity index 100% rename from .ci/es-snapshots/Jenkinsfile_verify_es.groovy rename to .ci/es-snapshots/Jenkinsfile_verify_es From b102d87acec1f3ac219fccb28d4a709012fb9b9c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 18 Dec 2019 11:59:48 -0500 Subject: [PATCH 12/27] Update es snapshot job names --- .ci/es-snapshots/Jenkinsfile_build_es | 2 +- .ci/es-snapshots/Jenkinsfile_trigger_build_es | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 2445ddc56e990e..76c9c7384cd447 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -74,7 +74,7 @@ timeout(time: 120, unit: 'MINUTES') { build( propagate: false, wait: false, - job: 'es-snapshot-verification', + job: 'elasticsearch / snapshots # verify', parameters: [ string(name: 'branch_specifier', value: branch_specifier), string(name: 'SNAPSHOT_VERSION', value: VERSION), diff --git a/.ci/es-snapshots/Jenkinsfile_trigger_build_es b/.ci/es-snapshots/Jenkinsfile_trigger_build_es index a765fde511ff24..24e973f1888767 100644 --- a/.ci/es-snapshots/Jenkinsfile_trigger_build_es +++ b/.ci/es-snapshots/Jenkinsfile_trigger_build_es @@ -10,7 +10,7 @@ branches.each { branch -> build( propagate: false, wait: false, - job: 'build-es', + job: 'elasticsearch / snapshots # build', parameters: [ string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO string(name: 'ES_BRANCH', value: branch), From c23b6b37d06bc2950b26b78584ae983a10a6a1ff Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 18 Dec 2019 12:03:16 -0500 Subject: [PATCH 13/27] Update job names again --- .ci/es-snapshots/Jenkinsfile_build_es | 2 +- .ci/es-snapshots/Jenkinsfile_trigger_build_es | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 76c9c7384cd447..678ec44aba6fba 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -74,7 +74,7 @@ timeout(time: 120, unit: 'MINUTES') { build( propagate: false, wait: false, - job: 'elasticsearch / snapshots # verify', + job: 'elasticsearch+snapshots+verify', parameters: [ string(name: 'branch_specifier', value: branch_specifier), string(name: 'SNAPSHOT_VERSION', value: VERSION), diff --git a/.ci/es-snapshots/Jenkinsfile_trigger_build_es b/.ci/es-snapshots/Jenkinsfile_trigger_build_es index 24e973f1888767..6c3f8bc535cbe1 100644 --- a/.ci/es-snapshots/Jenkinsfile_trigger_build_es +++ b/.ci/es-snapshots/Jenkinsfile_trigger_build_es @@ -10,7 +10,7 @@ branches.each { branch -> build( propagate: false, wait: false, - job: 'elasticsearch / snapshots # build', + job: 'elasticsearch+snapshots+build', parameters: [ string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO string(name: 'ES_BRANCH', value: branch), From 546fe69fd9db0d57b3d94827d2b289875ef56bdf Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 18 Dec 2019 14:24:37 -0500 Subject: [PATCH 14/27] Remove console.log thats breaking a test --- packages/kbn-es/src/artifact.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-es/src/artifact.js b/packages/kbn-es/src/artifact.js index cb48c2d4a35b0a..8d1a3f513e64b9 100644 --- a/packages/kbn-es/src/artifact.js +++ b/packages/kbn-es/src/artifact.js @@ -97,7 +97,6 @@ exports.Artifact = class Artifact { const customSnapshotArtifactSpec = await resolveCustomSnapshotUrl(urlVersion, license); if (customSnapshotArtifactSpec) { - console.log(customSnapshotArtifactSpec); return new Artifact(customSnapshotArtifactSpec, log); } From ede0cb621f57f02e614000d385675fe149f678f6 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 18 Dec 2019 18:03:00 -0500 Subject: [PATCH 15/27] Switch to specify custom manifest url instead of custom snapshot url --- .ci/es-snapshots/Jenkinsfile_build_es | 2 ++ .ci/es-snapshots/Jenkinsfile_verify_es | 11 +++++++---- packages/kbn-es/src/custom_snapshots.js | 18 +++++++++--------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 678ec44aba6fba..000c1bf4f7c37c 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -53,6 +53,7 @@ timeout(time: 120, unit: 'MINUTES') { sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' + // TODO remove all currently unused params? def manifest = [ bucket: "kibana-ci-es-snapshots/${DESTINATION}".toString(), branch: ES_BRANCH, @@ -67,6 +68,7 @@ timeout(time: 120, unit: 'MINUTES') { upload(DESTINATION, '*.*') + // Remove this? I like that there's an easy way to use it without having to look it up sh "cp manifest.json manifest-latest.json" upload(VERSION, 'manifest-latest.json') } diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index 21a29cddc660aa..369376ce8b30e0 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -7,8 +7,9 @@ kibanaLibrary.load() def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION ?: "8.0.0" def SNAPSHOT_ID = params.SNAPSHOT_ID ?: "20191216-202229_c1d075a7dab" -def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" -def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" +// def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" +// def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" +def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" timeout(time: 120, unit: 'MINUTES') { timestamps { @@ -16,7 +17,8 @@ timeout(time: 120, unit: 'MINUTES') { catchError { parallel([ oss: { - withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { + // withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { + withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { parallel([ 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), // TODO we just need to run integration tests from intake? 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ @@ -37,7 +39,8 @@ timeout(time: 120, unit: 'MINUTES') { } }, xpack: { - withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { + // withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { + withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { parallel([ 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), // TODO we just need to run integration tests from intake? 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ diff --git a/packages/kbn-es/src/custom_snapshots.js b/packages/kbn-es/src/custom_snapshots.js index b6de4e89064f4e..671a98c3e83490 100644 --- a/packages/kbn-es/src/custom_snapshots.js +++ b/packages/kbn-es/src/custom_snapshots.js @@ -55,15 +55,15 @@ async function getSnapshotManifest(urlVersion, license) { const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); const desiredLicense = license === 'oss' ? 'oss' : 'default'; - let { abc, resp, json } = await fetchSnapshotManifest( - `https://storage.googleapis.com/kibana-ci-es-snapshots/${desiredVersion}/manifest-latest${ - shouldUseUnverifiedSnapshot() ? '' : '-verified' - }.json` - ); - if (!shouldUseUnverifiedSnapshot() && resp.status === 404) { - ({ abc, resp, json } = await fetchSnapshotManifest( - `https://storage.googleapis.com/kibana-ci-es-snapshots-permanent/${desiredVersion}/manifest.json` - )); + const customManifestUrl = process.env.ES_SNAPSHOT_MANIFEST; + const primaryManifestUrl = `https://storage.googleapis.com/kibana-ci-es-snapshots/${desiredVersion}/manifest-latest${ + shouldUseUnverifiedSnapshot() ? '' : '-verified' + }.json`; + const secondaryManifestUrl = `https://storage.googleapis.com/kibana-ci-es-snapshots-permanent/${desiredVersion}/manifest.json`; + + let { abc, resp, json } = await fetchSnapshotManifest(customManifestUrl || primaryManifestUrl); + if (!customManifestUrl && !shouldUseUnverifiedSnapshot() && resp.status === 404) { + ({ abc, resp, json } = await fetchSnapshotManifest(secondaryManifestUrl)); } if (resp.status === 404) { From 82163330c0a463463f8b72143d7222d8cefa36ef Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 19 Dec 2019 16:17:05 -0500 Subject: [PATCH 16/27] Remove release-manager snapshot handling, add some tests for new snapshot handling --- packages/kbn-es/src/artifact.js | 159 +++++++++++------------- packages/kbn-es/src/artifact.test.js | 98 +++++++++++++++ packages/kbn-es/src/custom_snapshots.js | 76 ++--------- 3 files changed, 180 insertions(+), 153 deletions(-) create mode 100644 packages/kbn-es/src/artifact.test.js diff --git a/packages/kbn-es/src/artifact.js b/packages/kbn-es/src/artifact.js index 8d1a3f513e64b9..beb1e0044957c7 100644 --- a/packages/kbn-es/src/artifact.js +++ b/packages/kbn-es/src/artifact.js @@ -27,16 +27,14 @@ const { createHash } = require('crypto'); const path = require('path'); const asyncPipeline = promisify(pipeline); -const V1_VERSIONS_API = 'https://artifacts-api.elastic.co/v1/versions'; +const DAILY_SNAPSHOTS_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots'; +const PERMANENT_SNAPSHOTS_BASE_URL = + 'https://storage.googleapis.com/kibana-ci-es-snapshots-permanent'; const { cache } = require('./utils'); const { resolveCustomSnapshotUrl } = require('./custom_snapshots'); const { createCliError, isCliError } = require('./errors'); -const TEST_ES_SNAPSHOT_VERSION = process.env.TEST_ES_SNAPSHOT_VERSION - ? process.env.TEST_ES_SNAPSHOT_VERSION - : 'latest'; - function getChecksumType(checksumUrl) { if (checksumUrl.endsWith('.sha512')) { return 'sha512'; @@ -45,20 +43,6 @@ function getChecksumType(checksumUrl) { throw new Error(`unable to determine checksum type: ${checksumUrl}`); } -function getPlatform(key) { - if (key.includes('-linux-')) { - return 'linux'; - } - - if (key.includes('-windows-')) { - return 'win32'; - } - - if (key.includes('-darwin-')) { - return 'darwin'; - } -} - function headersToString(headers, indent = '') { return [...headers.entries()].reduce( (acc, [key, value]) => `${acc}\n${indent}${key}: ${value}`, @@ -85,6 +69,75 @@ async function retry(log, fn) { return await doAttempt(1); } +// Setting this flag provides an easy way to run the latest un-promoted snapshot without having to look it up +function shouldUseUnverifiedSnapshot() { + return !!process.env.KBN_ES_SNAPSHOT_USE_UNVERIFIED; +} + +async function fetchSnapshotManifest(url, log) { + log.info('Downloading snapshot manifest from %s', chalk.bold(url)); + + const abc = new AbortController(); + const resp = await retry(log, async () => await fetch(url, { signal: abc.signal })); + const json = await resp.text(); + + return { abc, resp, json }; +} + +async function getArtifactSpecForSnapshot(urlVersion, license, log) { + const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); + const desiredLicense = license === 'oss' ? 'oss' : 'default'; + + const customManifestUrl = process.env.ES_SNAPSHOT_MANIFEST; + const primaryManifestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}/manifest-latest${ + shouldUseUnverifiedSnapshot() ? '' : '-verified' + }.json`; + const secondaryManifestUrl = `${PERMANENT_SNAPSHOTS_BASE_URL}/${desiredVersion}/manifest.json`; + + let { abc, resp, json } = await fetchSnapshotManifest( + customManifestUrl || primaryManifestUrl, + log + ); + + if (!customManifestUrl && !shouldUseUnverifiedSnapshot() && resp.status === 404) { + log.info('Daily snapshot manifest not found, falling back to permanent manifest'); + ({ abc, resp, json } = await fetchSnapshotManifest(secondaryManifestUrl, log)); + } + + if (resp.status === 404) { + abc.abort(); + throw createCliError(`Snapshots for ${desiredVersion} are not available`); + } + + if (!resp.ok) { + abc.abort(); + throw new Error(`Unable to read snapshot manifest: ${resp.statusText}\n ${json}`); + } + + const manifest = JSON.parse(json); + + const platform = process.platform === 'win32' ? 'windows' : process.platform; + const archive = manifest.archives.find( + archive => + archive.version === desiredVersion && + archive.platform === platform && + archive.license === desiredLicense + ); + + if (!archive) { + throw createCliError( + `Snapshots for ${desiredVersion} are available, but couldn't find an artifact in the manifest for [${desiredVersion}, ${desiredLicense}, ${platform}]` + ); + } + + return { + url: archive.url, + checksumUrl: archive.url + '.sha512', + checksumType: 'sha512', + filename: archive.filename, + }; +} + exports.Artifact = class Artifact { /** * Fetch an Artifact from the Artifact API for a license level and version @@ -95,76 +148,12 @@ exports.Artifact = class Artifact { static async getSnapshot(license, version, log) { const urlVersion = `${encodeURIComponent(version)}-SNAPSHOT`; - const customSnapshotArtifactSpec = await resolveCustomSnapshotUrl(urlVersion, license); + const customSnapshotArtifactSpec = resolveCustomSnapshotUrl(urlVersion, license); if (customSnapshotArtifactSpec) { return new Artifact(customSnapshotArtifactSpec, log); } - const urlBuild = encodeURIComponent(TEST_ES_SNAPSHOT_VERSION); - const url = `${V1_VERSIONS_API}/${urlVersion}/builds/${urlBuild}/projects/elasticsearch`; - - const json = await retry(log, async () => { - log.info('downloading artifact info from %s', chalk.bold(url)); - - const abc = new AbortController(); - const resp = await fetch(url, { signal: abc.signal }); - const json = await resp.text(); - - if (resp.status === 404) { - abc.abort(); - throw createCliError( - `Snapshots for ${version}/${TEST_ES_SNAPSHOT_VERSION} are not available` - ); - } - - if (!resp.ok) { - abc.abort(); - throw new Error(`Unable to read artifact info from ${url}: ${resp.statusText}\n ${json}`); - } - - return json; - }); - - // parse the api response into an array of Artifact objects - const { - project: { packages: artifactInfoMap }, - } = JSON.parse(json); - const filenames = Object.keys(artifactInfoMap); - const hasNoJdkVersions = filenames.some(filename => filename.includes('-no-jdk-')); - const artifactSpecs = filenames.map(filename => ({ - filename, - url: artifactInfoMap[filename].url, - checksumUrl: artifactInfoMap[filename].sha_url, - checksumType: getChecksumType(artifactInfoMap[filename].sha_url), - type: artifactInfoMap[filename].type, - isOss: filename.includes('-oss-'), - platform: getPlatform(filename), - jdkRequired: hasNoJdkVersions ? filename.includes('-no-jdk-') : true, - })); - - // pick the artifact we are going to use for this license/version combo - const reqOss = license === 'oss'; - const reqPlatform = artifactSpecs.some(a => a.platform !== undefined) - ? process.platform - : undefined; - const reqJdkRequired = hasNoJdkVersions ? false : true; - const reqType = process.platform === 'win32' ? 'zip' : 'tar'; - - const artifactSpec = artifactSpecs.find( - spec => - spec.isOss === reqOss && - spec.type === reqType && - spec.platform === reqPlatform && - spec.jdkRequired === reqJdkRequired - ); - - if (!artifactSpec) { - throw new Error( - `Unable to determine artifact for license [${license}] and version [${version}]\n` + - ` options: ${filenames.join(',')}` - ); - } - + const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log); return new Artifact(artifactSpec, log); } diff --git a/packages/kbn-es/src/artifact.test.js b/packages/kbn-es/src/artifact.test.js new file mode 100644 index 00000000000000..dcd6c2e8ebf51f --- /dev/null +++ b/packages/kbn-es/src/artifact.test.js @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; +jest.mock('node-fetch'); +import fetch from 'node-fetch'; +const { Response } = jest.requireActual('node-fetch'); + +import { Artifact } from './artifact'; + +const log = new ToolingLog(); +let MOCKS; + +const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform; +const MOCK_VERSION = 'test-version'; +const MOCK_URL = 'http://127.0.0.1:12345'; +const MOCK_FILENAME = 'test-filename'; + +const createArchive = (params = {}) => { + const license = params.license || 'default'; + + return { + license: 'default', + version: MOCK_VERSION, + url: MOCK_URL + `/${license}`, + platform: PLATFORM, + filename: MOCK_FILENAME + `.${license}`, + ...params, + }; +}; + +const mockFetch = mock => + fetch.mockReturnValue(Promise.resolve(new Response(JSON.stringify(mock)))); + +beforeEach(() => { + jest.resetAllMocks(); + + MOCKS = { + valid: { + archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })], + }, + }; +}); + +const artifactTest = (requestedLicense, expectedLicense) => { + return async () => { + const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log); + expect(fetch).toHaveBeenCalledTimes(1); + expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`); + expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`); + expect(artifact.getChecksumType()).toEqual('sha512'); + expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `.${expectedLicense}`); + }; +}; + +describe('Artifact', () => { + describe('getSnapshot()', () => { + describe('with default snapshot', () => { + beforeEach(() => { + mockFetch(MOCKS.valid); + }); + + it('should return artifact metadata for an oss artifact', artifactTest('oss', 'oss')); + + it( + 'should return artifact metadata for a default artifact', + artifactTest('default', 'default') + ); + + it( + 'should default to default license with anything other than "oss"', + artifactTest('INVALID_LICENSE', 'default') + ); + + it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { + await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( + "couldn't find an artifact" + ); + }); + }); + }); +}); diff --git a/packages/kbn-es/src/custom_snapshots.js b/packages/kbn-es/src/custom_snapshots.js index 671a98c3e83490..c6b00f77f0a881 100644 --- a/packages/kbn-es/src/custom_snapshots.js +++ b/packages/kbn-es/src/custom_snapshots.js @@ -16,25 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -const AbortController = require('abort-controller'); -const fetch = require('node-fetch'); -const { basename } = require('path'); -const { createCliError } = require('./errors'); +const { basename } = require('path'); function isVersionFlag(a) { return a.startsWith('--version'); } -function shouldUseUnverifiedSnapshot() { - return !!process.env.KBN_ES_SNAPSHOT_USE_UNVERIFIED; -} - function getCustomSnapshotUrl() { // force use of manually created snapshots until ReindexPutMappings fix - if (!process.env.KBN_ES_SNAPSHOT_URL && !process.argv.some(isVersionFlag)) { + if ( + !process.env.ES_SNAPSHOT_MANIFEST && + !process.env.KBN_ES_SNAPSHOT_URL && + !process.argv.some(isVersionFlag) + ) { + // return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}'; return; - return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}'; } if (process.env.KBN_ES_SNAPSHOT_URL && process.env.KBN_ES_SNAPSHOT_URL !== 'false') { @@ -42,67 +39,10 @@ function getCustomSnapshotUrl() { } } -// TODO all of this stuff will probably move somewhere else -async function fetchSnapshotManifest(url) { - const abc = new AbortController(); - const resp = await fetch(url, { signal: abc.signal }); - const json = await resp.text(); - - return { abc, resp, json }; -} - -async function getSnapshotManifest(urlVersion, license) { - const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); - const desiredLicense = license === 'oss' ? 'oss' : 'default'; - - const customManifestUrl = process.env.ES_SNAPSHOT_MANIFEST; - const primaryManifestUrl = `https://storage.googleapis.com/kibana-ci-es-snapshots/${desiredVersion}/manifest-latest${ - shouldUseUnverifiedSnapshot() ? '' : '-verified' - }.json`; - const secondaryManifestUrl = `https://storage.googleapis.com/kibana-ci-es-snapshots-permanent/${desiredVersion}/manifest.json`; - - let { abc, resp, json } = await fetchSnapshotManifest(customManifestUrl || primaryManifestUrl); - if (!customManifestUrl && !shouldUseUnverifiedSnapshot() && resp.status === 404) { - ({ abc, resp, json } = await fetchSnapshotManifest(secondaryManifestUrl)); - } - - if (resp.status === 404) { - abc.abort(); - throw createCliError(`Snapshots for ${desiredVersion} are not available`); - } - - if (!resp.ok) { - abc.abort(); - throw new Error(`Unable to read artifact info: ${resp.statusText}\n ${json}`); - } - - const manifest = JSON.parse(json); - - const platform = process.platform === 'win32' ? 'windows' : process.platform; - const archive = manifest.archives.find( - archive => - archive.version === desiredVersion && - archive.platform === platform && - archive.license === desiredLicense - ); - - return { - url: archive.url, - checksumUrl: archive.url + '.sha512', - checksumType: 'sha512', - filename: archive.filename, - }; -} - -async function resolveCustomSnapshotUrl(urlVersion, license) { +function resolveCustomSnapshotUrl(urlVersion, license) { const customSnapshotUrl = getCustomSnapshotUrl(); if (!customSnapshotUrl) { - const manifest = getSnapshotManifest(urlVersion, license); - if (manifest) { - return manifest; - } - return; } From 64a40d5cd66c84984fe48d1a5a244f188e44ade1 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 19 Dec 2019 16:48:05 -0500 Subject: [PATCH 17/27] Cleanup --- .ci/es-snapshots/Jenkinsfile_build_es | 14 +++- .ci/es-snapshots/Jenkinsfile_verify_es | 92 ++++++++++++-------------- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 000c1bf4f7c37c..66c6e4dcd07669 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -1,9 +1,17 @@ #!/bin/groovy +// This job effectively has two SCM configurations: +// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job +// one for elasticsearch, used to check out the elasticsearch source before building it + +// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same +// 'branch_specifier' is for kibana / the job itself +// ES_BRANCH is for elasticsearch + library 'kibana-pipeline-library' kibanaLibrary.load() -def ES_BRANCH = params.ES_BRANCH ?: 'master' // TODO +def ES_BRANCH = params.ES_BRANCH ES_BRANCH = ES_BRANCH != 'master' ? ES_BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists if (!ES_BRANCH) { @@ -23,7 +31,6 @@ timeout(time: 120, unit: 'MINUTES') { def GIT_COMMIT = scmVars.GIT_COMMIT def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() - sh 'rm -rf to-archive || true' // TODO remove buildArchives('to-archive') dir('to-archive') { @@ -32,6 +39,7 @@ timeout(time: 120, unit: 'MINUTES') { def filesRaw = sh(script: "ls -1", returnStdout: true).trim() def files = filesRaw.split("\n").collect { filename -> + // Filename examples // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz def parts = filename.replace("elasticsearch-oss", "oss").split("-") @@ -120,7 +128,7 @@ def upload(destination, pattern) { def buildArchives(destination) { def props = readProperties file: '.ci/java-versions.properties' withEnv([ - "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", // Probably won't even need this if this gets wrapped in runbld + "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", // these jenkins env vars trigger some automation in ES that we don't want "BUILD_NUMBER=", diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index 369376ce8b30e0..e69433c93e9992 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -3,62 +3,56 @@ library 'kibana-pipeline-library' kibanaLibrary.load() -// TODO finalize these -def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION ?: "8.0.0" -def SNAPSHOT_ID = params.SNAPSHOT_ID ?: "20191216-202229_c1d075a7dab" +def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION +def SNAPSHOT_ID = params.SNAPSHOT_ID + +if (!SNAPSHOT_VERSION) { + error "Parameter SNAPSHOT_VERSION must be specified" +} + +if (!SNAPSHOT_ID) { + error "Parameter SNAPSHOT_ID must be specified" +} -// def ES_OSS_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-oss-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" -// def ES_XPACK_SNAPSHOT = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/elasticsearch-${SNAPSHOT_VERSION}-SNAPSHOT-linux-x86_64.tar.gz" def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { catchError { - parallel([ - oss: { - // withEnv(["KBN_ES_SNAPSHOT_URL=${ES_OSS_SNAPSHOT}"]) { - withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { - parallel([ - 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), // TODO we just need to run integration tests from intake? - 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), - 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - ]), - ]) - } - }, - xpack: { - // withEnv(["KBN_ES_SNAPSHOT_URL=${ES_XPACK_SNAPSHOT}"]) { - withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { - parallel([ - 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), // TODO we just need to run integration tests from intake? - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - ]), - ]) - } - } - ]) + withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { + parallel([ + // TODO we just need to run integration tests from intake? + 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), + 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), + 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), + 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), + 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), + 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), + 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), + 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), + 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), + 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), + 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), + 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), + 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), + 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), + ]), + 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), + 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), + 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), + 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), + 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), + 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), + 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), + 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), + 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), + 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), + ]), + ]) + } promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) } From 23e0e9f1698b5f454c5db05790fe75c4fb700808 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 20 Dec 2019 10:25:16 -0500 Subject: [PATCH 18/27] Add paramaters to es snapshot build names --- .ci/es-snapshots/Jenkinsfile_build_es | 2 ++ .ci/es-snapshots/Jenkinsfile_verify_es | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 66c6e4dcd07669..a3525b87905ffe 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -18,6 +18,8 @@ if (!ES_BRANCH) { error "Parameter 'ES_BRANCH' must be specified." } +currentBuild.displayName += " ES ${params.ES_BRANCH} / Kibana ${params.branch_specifier}" + timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index e69433c93e9992..d54664df6e5f0f 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -14,6 +14,8 @@ if (!SNAPSHOT_ID) { error "Parameter SNAPSHOT_ID must be specified" } +currentBuild.displayName += "ES ${SNAPSHOT_VERSION} / Kibana ${params.branch_specifier}" + def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" timeout(time: 120, unit: 'MINUTES') { From be34c22c68afd1c5c69695db1a5453a0beaf4460 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 20 Dec 2019 10:29:12 -0500 Subject: [PATCH 19/27] Build display name too long, try something shorter --- .ci/es-snapshots/Jenkinsfile_build_es | 3 ++- .ci/es-snapshots/Jenkinsfile_verify_es | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index a3525b87905ffe..bee1ebf8d214a9 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -18,7 +18,8 @@ if (!ES_BRANCH) { error "Parameter 'ES_BRANCH' must be specified." } -currentBuild.displayName += " ES ${params.ES_BRANCH} / Kibana ${params.branch_specifier}" +currentBuild.displayName += " ${params.ES_BRANCH}" +currentBuild.description = "ES: ${params.ES_BRANCH}\nKibana: ${params.branch_specifier}" timeout(time: 120, unit: 'MINUTES') { timestamps { diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index d54664df6e5f0f..6fd8211025a29a 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -14,7 +14,8 @@ if (!SNAPSHOT_ID) { error "Parameter SNAPSHOT_ID must be specified" } -currentBuild.displayName += "ES ${SNAPSHOT_VERSION} / Kibana ${params.branch_specifier}" +currentBuild.displayName += " ${SNAPSHOT_VERSION}" +currentBuild.description = "ES: ${SNAPSHOT_VERSION}\nKibana: ${params.branch_specifier}" def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" From 4ace1cff25d4c2f0efb93b005eb299819b8cba0e Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 20 Dec 2019 11:28:43 -0500 Subject: [PATCH 20/27] Add more unit tests, do some cleanup, load branches from tracked branches yaml --- .ci/es-snapshots/Jenkinsfile_build_es | 23 ++--- .ci/es-snapshots/Jenkinsfile_trigger_build_es | 10 +-- .ci/es-snapshots/Jenkinsfile_verify_es | 4 +- packages/kbn-es/src/artifact.test.js | 84 ++++++++++++++++++- 4 files changed, 99 insertions(+), 22 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index bee1ebf8d214a9..cf369fbdf751e8 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -18,8 +18,8 @@ if (!ES_BRANCH) { error "Parameter 'ES_BRANCH' must be specified." } -currentBuild.displayName += " ${params.ES_BRANCH}" -currentBuild.description = "ES: ${params.ES_BRANCH}\nKibana: ${params.branch_specifier}" +currentBuild.displayName += " - ${params.ES_BRANCH}" +currentBuild.description = "ES: ${params.ES_BRANCH}
Kibana: ${params.branch_specifier}" timeout(time: 120, unit: 'MINUTES') { timestamps { @@ -56,7 +56,7 @@ timeout(time: 120, unit: 'MINUTES') { checksum: filename + '.sha512', url: "https://storage.googleapis.com/kibana-ci-es-snapshots/${DESTINATION}/${filename}".toString(), version: parts[1], - platform: parts.size() >= 5 ? parts[3] : '', + platform: parts.size() >= 5 ? parts[3] : 'no-jdk', architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', license: parts[0] == 'oss' ? 'oss' : 'default', ] @@ -131,14 +131,15 @@ def upload(destination, pattern) { def buildArchives(destination) { def props = readProperties file: '.ci/java-versions.properties' withEnv([ - "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", - - // these jenkins env vars trigger some automation in ES that we don't want - "BUILD_NUMBER=", - "JENKINS_URL=", - "BUILD_URL=", - "JOB_NAME=", - "NODE_NAME=", + // Select the correct JDK for this branch + "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", + + // these jenkins env vars trigger some automation in ES that we don't want + "BUILD_NUMBER=", + "JENKINS_URL=", + "BUILD_URL=", + "JOB_NAME=", + "NODE_NAME=", ]) { sh """ ./gradlew -p distribution/archives assemble --parallel diff --git a/.ci/es-snapshots/Jenkinsfile_trigger_build_es b/.ci/es-snapshots/Jenkinsfile_trigger_build_es index 6c3f8bc535cbe1..d6597d86688be6 100644 --- a/.ci/es-snapshots/Jenkinsfile_trigger_build_es +++ b/.ci/es-snapshots/Jenkinsfile_trigger_build_es @@ -1,10 +1,10 @@ #!/bin/groovy -// TODO is there a good place to grab a list of these? -def branches = [ - 'master', - // '7.4', -] +if (!params.branches_yaml) { + error "'branches_yaml' parameter must be specified" +} + +def branches = readYaml text: params.branches_yaml branches.each { branch -> build( diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index 6fd8211025a29a..a0fbc0180e93db 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -14,8 +14,8 @@ if (!SNAPSHOT_ID) { error "Parameter SNAPSHOT_ID must be specified" } -currentBuild.displayName += " ${SNAPSHOT_VERSION}" -currentBuild.description = "ES: ${SNAPSHOT_VERSION}\nKibana: ${params.branch_specifier}" +currentBuild.displayName += " - ${SNAPSHOT_VERSION}" +currentBuild.description = "ES: ${SNAPSHOT_VERSION}
Kibana: ${params.branch_specifier}" def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" diff --git a/packages/kbn-es/src/artifact.test.js b/packages/kbn-es/src/artifact.test.js index dcd6c2e8ebf51f..c63bd716f6eedc 100644 --- a/packages/kbn-es/src/artifact.test.js +++ b/packages/kbn-es/src/artifact.test.js @@ -32,6 +32,10 @@ const MOCK_VERSION = 'test-version'; const MOCK_URL = 'http://127.0.0.1:12345'; const MOCK_FILENAME = 'test-filename'; +const DAILY_SNAPSHOT_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots'; +const PERMANENT_SNAPSHOT_BASE_URL = + 'https://storage.googleapis.com/kibana-ci-es-snapshots-permanent'; + const createArchive = (params = {}) => { const license = params.license || 'default'; @@ -58,10 +62,18 @@ beforeEach(() => { }; }); -const artifactTest = (requestedLicense, expectedLicense) => { +const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => { return async () => { const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log); - expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled); + expect(fetch.mock.calls[0][0]).toEqual( + `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json` + ); + if (fetchTimesCalled === 2) { + expect(fetch.mock.calls[1][0]).toEqual( + `${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json` + ); + } expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`); expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`); expect(artifact.getChecksumType()).toEqual('sha512'); @@ -76,10 +88,10 @@ describe('Artifact', () => { mockFetch(MOCKS.valid); }); - it('should return artifact metadata for an oss artifact', artifactTest('oss', 'oss')); + it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss')); it( - 'should return artifact metadata for a default artifact', + 'should return artifact metadata for a daily default artifact', artifactTest('default', 'default') ); @@ -94,5 +106,69 @@ describe('Artifact', () => { ); }); }); + + describe('with missing default snapshot', () => { + beforeEach(() => { + fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); + mockFetch(MOCKS.valid); + }); + + it( + 'should return artifact metadata for a permanent oss artifact', + artifactTest('oss', 'oss', 2) + ); + + it( + 'should return artifact metadata for a permanent default artifact', + artifactTest('default', 'default', 2) + ); + + it( + 'should default to default license with anything other than "oss"', + artifactTest('INVALID_LICENSE', 'default', 2) + ); + + it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { + await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( + "couldn't find an artifact" + ); + }); + }); + + describe('with custom snapshot manifest URL', () => { + const CUSTOM_URL = 'http://www.creedthoughts.gov.www/creedthoughts'; + + beforeEach(() => { + process.env.ES_SNAPSHOT_MANIFEST = CUSTOM_URL; + mockFetch(MOCKS.valid); + }); + + it('should use the custom URL when looking for a snapshot', async () => { + await Artifact.getSnapshot('oss', MOCK_VERSION, log); + expect(fetch.mock.calls[0][0]).toEqual(CUSTOM_URL); + }); + + afterEach(() => { + delete process.env.ES_SNAPSHOT_MANIFEST; + }); + }); + + describe('with latest unverified snapshot', () => { + beforeEach(() => { + process.env.KBN_ES_SNAPSHOT_USE_UNVERIFIED = 1; + mockFetch(MOCKS.valid); + }); + + it('should use the daily unverified URL when looking for a snapshot', async () => { + await Artifact.getSnapshot('oss', MOCK_VERSION, log); + expect(fetch.mock.calls[0][0]).toEqual( + `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json` + ); + }); + + afterEach(() => { + delete process.env.KBN_ES_SNAPSHOT_USE_UNVERIFIED; + }); + }); }); }); From 9edfc4f70a3de9de5686d26d69ec44f302615457 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 2 Jan 2020 14:57:22 -0500 Subject: [PATCH 21/27] Add ability to promote ES snapshots without running verification --- .ci/es-snapshots/Jenkinsfile_build_es | 36 +++++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index cf369fbdf751e8..8d2f49acf9b8dd 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -21,6 +21,8 @@ if (!ES_BRANCH) { currentBuild.displayName += " - ${params.ES_BRANCH}" currentBuild.description = "ES: ${params.ES_BRANCH}
Kibana: ${params.branch_specifier}" +def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION + timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { @@ -79,21 +81,33 @@ timeout(time: 120, unit: 'MINUTES') { upload(DESTINATION, '*.*') - // Remove this? I like that there's an easy way to use it without having to look it up + // TODO Remove this? I like that there's an easy way to use it without having to look it up sh "cp manifest.json manifest-latest.json" upload(VERSION, 'manifest-latest.json') } - build( - propagate: false, - wait: false, - job: 'elasticsearch+snapshots+verify', - parameters: [ - string(name: 'branch_specifier', value: branch_specifier), - string(name: 'SNAPSHOT_VERSION', value: VERSION), - string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), - ] - ) + if (PROMOTE_WITHOUT_VERIFY) { + esSnapshots.promote(VERSION, SNAPSHOT_ID) + + emailext( + // to: 'build-kibana@elastic.co', + to: 'brian.seeders@elastic.co', // TODO change this back + subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}", + body: '${SCRIPT,template="groovy-html.template"}', + mimeType: 'text/html', + ) + } else { + build( + propagate: false, + wait: false, + job: 'elasticsearch+snapshots+verify', + parameters: [ + string(name: 'branch_specifier', value: branch_specifier), + string(name: 'SNAPSHOT_VERSION', value: VERSION), + string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), + ] + ) + } } kibanaPipeline.sendMail() From a1a496bd580f6ebb066c38ce7d2f8da76a2f9090 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 2 Jan 2020 16:19:16 -0500 Subject: [PATCH 22/27] Move daily es snapshots bucket --- .ci/es-snapshots/Jenkinsfile_build_es | 6 +++--- .ci/es-snapshots/Jenkinsfile_verify_es | 2 +- packages/kbn-es/src/artifact.js | 2 +- packages/kbn-es/src/artifact.test.js | 2 +- vars/esSnapshots.groovy | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 8d2f49acf9b8dd..5411fe6d6f7867 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -56,7 +56,7 @@ timeout(time: 120, unit: 'MINUTES') { return [ filename: filename, checksum: filename + '.sha512', - url: "https://storage.googleapis.com/kibana-ci-es-snapshots/${DESTINATION}/${filename}".toString(), + url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(), version: parts[1], platform: parts.size() >= 5 ? parts[3] : 'no-jdk', architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', @@ -68,7 +68,7 @@ timeout(time: 120, unit: 'MINUTES') { // TODO remove all currently unused params? def manifest = [ - bucket: "kibana-ci-es-snapshots/${DESTINATION}".toString(), + bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(), branch: ES_BRANCH, sha: GIT_COMMIT, sha_short: GIT_COMMIT_SHORT, @@ -135,7 +135,7 @@ def checkoutEs(branch) { def upload(destination, pattern) { return googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://kibana-ci-es-snapshots/${destination}", + bucket: "gs://kibana-ci-es-snapshots-daily/${destination}", pattern: pattern, sharedPublicly: false, showInline: false, diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index a0fbc0180e93db..3d5ec75fa0e72a 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -17,7 +17,7 @@ if (!SNAPSHOT_ID) { currentBuild.displayName += " - ${SNAPSHOT_VERSION}" currentBuild.description = "ES: ${SNAPSHOT_VERSION}
Kibana: ${params.branch_specifier}" -def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" +def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" timeout(time: 120, unit: 'MINUTES') { timestamps { diff --git a/packages/kbn-es/src/artifact.js b/packages/kbn-es/src/artifact.js index beb1e0044957c7..9ea78386269d93 100644 --- a/packages/kbn-es/src/artifact.js +++ b/packages/kbn-es/src/artifact.js @@ -27,7 +27,7 @@ const { createHash } = require('crypto'); const path = require('path'); const asyncPipeline = promisify(pipeline); -const DAILY_SNAPSHOTS_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots'; +const DAILY_SNAPSHOTS_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots-daily'; const PERMANENT_SNAPSHOTS_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots-permanent'; diff --git a/packages/kbn-es/src/artifact.test.js b/packages/kbn-es/src/artifact.test.js index c63bd716f6eedc..bff64da3ca4925 100644 --- a/packages/kbn-es/src/artifact.test.js +++ b/packages/kbn-es/src/artifact.test.js @@ -32,7 +32,7 @@ const MOCK_VERSION = 'test-version'; const MOCK_URL = 'http://127.0.0.1:12345'; const MOCK_FILENAME = 'test-filename'; -const DAILY_SNAPSHOT_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots'; +const DAILY_SNAPSHOT_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots-daily'; const PERMANENT_SNAPSHOT_BASE_URL = 'https://storage.googleapis.com/kibana-ci-es-snapshots-permanent'; diff --git a/vars/esSnapshots.groovy b/vars/esSnapshots.groovy index e3b2844e8ede9b..884fbcdb17aebb 100644 --- a/vars/esSnapshots.groovy +++ b/vars/esSnapshots.groovy @@ -1,6 +1,6 @@ def promote(snapshotVersion, snapshotId) { def snapshotDestination = "${snapshotVersion}/archives/${snapshotId}" - def MANIFEST_URL = "https://storage.googleapis.com/kibana-ci-es-snapshots/${snapshotDestination}/manifest.json" + def MANIFEST_URL = "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${snapshotDestination}/manifest.json" dir('verified-manifest') { def verifiedSnapshotFilename = 'manifest-latest-verified.json' @@ -12,7 +12,7 @@ def promote(snapshotVersion, snapshotId) { googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://kibana-ci-es-snapshots/${snapshotVersion}", + bucket: "gs://kibana-ci-es-snapshots-daily/${snapshotVersion}", pattern: verifiedSnapshotFilename, sharedPublicly: false, showInline: false, @@ -24,7 +24,7 @@ def promote(snapshotVersion, snapshotId) { dir('transfer-to-permanent') { googleStorageDownload( credentialsId: 'kibana-ci-gcs-plugin', - bucketUri: "gs://kibana-ci-es-snapshots/${snapshotDestination}/*", + bucketUri: "gs://kibana-ci-es-snapshots-daily/${snapshotDestination}/*", localDirectory: '.', pathPrefix: snapshotDestination, ) @@ -32,7 +32,7 @@ def promote(snapshotVersion, snapshotId) { def manifestJson = readFile file: 'manifest.json' writeFile( file: 'manifest.json', - text: manifestJson.replace("kibana-ci-es-snapshots/${snapshotDestination}", "kibana-ci-es-snapshots-permanent/${snapshotVersion}") + text: manifestJson.replace("kibana-ci-es-snapshots-daily/${snapshotDestination}", "kibana-ci-es-snapshots-permanent/${snapshotVersion}") ) // Ideally we would have some delete logic here before uploading, From 9a4a661b369395067800813bf933a506d69c2a8d Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 11:36:52 -0500 Subject: [PATCH 23/27] Some cleanup and TODO completion --- .ci/es-snapshots/Jenkinsfile_build_es | 59 ++++++++++--------- .ci/es-snapshots/Jenkinsfile_trigger_build_es | 2 +- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 5411fe6d6f7867..a1873376c0cc9a 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -12,14 +12,13 @@ library 'kibana-pipeline-library' kibanaLibrary.load() def ES_BRANCH = params.ES_BRANCH -ES_BRANCH = ES_BRANCH != 'master' ? ES_BRANCH : 'c1d075a7dab901bc3c80dec55ef74cc0e64b87b9'; // Last known working 8.0.0 version, TODO: Remove once a verified snapshot exists if (!ES_BRANCH) { error "Parameter 'ES_BRANCH' must be specified." } -currentBuild.displayName += " - ${params.ES_BRANCH}" -currentBuild.description = "ES: ${params.ES_BRANCH}
Kibana: ${params.branch_specifier}" +currentBuild.displayName += " - ${ES_BRANCH}" +currentBuild.description = "ES: ${ES_BRANCH}
Kibana: ${params.branch_specifier}" def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION @@ -43,26 +42,34 @@ timeout(time: 120, unit: 'MINUTES') { def date = now.format("yyyyMMdd-HHmmss") def filesRaw = sh(script: "ls -1", returnStdout: true).trim() - def files = filesRaw.split("\n").collect { filename -> - // Filename examples - // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - def parts = filename.replace("elasticsearch-oss", "oss").split("-") - - VERSION = VERSION ?: parts[1] - SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" - DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" - - return [ - filename: filename, - checksum: filename + '.sha512', - url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(), - version: parts[1], - platform: parts.size() >= 5 ? parts[3] : 'no-jdk', - architecture: parts.size() >= 5 ? parts[4].split('\\.')[0] : '', - license: parts[0] == 'oss' ? 'oss' : 'default', - ] - } + def files = filesRaw + .split("\n") + .collect { filename -> + // Filename examples + // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz + def parts = filename.replace("elasticsearch-oss", "oss").split("-") + + // We don't care about the no-jdk artifact + if (parts.size() < 5) { + return null + } + + VERSION = VERSION ?: parts[1] + SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" + DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" + + return [ + filename: filename, + checksum: filename + '.sha512', + url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(), + version: parts[1], + platform: parts[3], + architecture: parts[4].split('\\.')[0], + license: parts[0] == 'oss' ? 'oss' : 'default', + ] + } + .filter { !!it } sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' @@ -81,7 +88,6 @@ timeout(time: 120, unit: 'MINUTES') { upload(DESTINATION, '*.*') - // TODO Remove this? I like that there's an easy way to use it without having to look it up sh "cp manifest.json manifest-latest.json" upload(VERSION, 'manifest-latest.json') } @@ -90,8 +96,7 @@ timeout(time: 120, unit: 'MINUTES') { esSnapshots.promote(VERSION, SNAPSHOT_ID) emailext( - // to: 'build-kibana@elastic.co', - to: 'brian.seeders@elastic.co', // TODO change this back + to: 'build-kibana@elastic.co', subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}", body: '${SCRIPT,template="groovy-html.template"}', mimeType: 'text/html', @@ -148,7 +153,7 @@ def buildArchives(destination) { // Select the correct JDK for this branch "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", - // these jenkins env vars trigger some automation in ES that we don't want + // These Jenkins env vars trigger some automation in ES that we don't want "BUILD_NUMBER=", "JENKINS_URL=", "BUILD_URL=", diff --git a/.ci/es-snapshots/Jenkinsfile_trigger_build_es b/.ci/es-snapshots/Jenkinsfile_trigger_build_es index d6597d86688be6..186917e9678245 100644 --- a/.ci/es-snapshots/Jenkinsfile_trigger_build_es +++ b/.ci/es-snapshots/Jenkinsfile_trigger_build_es @@ -12,7 +12,7 @@ branches.each { branch -> wait: false, job: 'elasticsearch+snapshots+build', parameters: [ - string(name: 'branch_specifier', value: 'es-snapshots-automation'), // TODO + string(name: 'branch_specifier', value: branch), string(name: 'ES_BRANCH', value: branch), ] ) From 1bff79e8f3ee0251510d815be9f9dc65778eaa4f Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 14:06:05 -0500 Subject: [PATCH 24/27] Fix list filter method --- .ci/es-snapshots/Jenkinsfile_build_es | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index a1873376c0cc9a..fbed42541a9372 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -69,7 +69,7 @@ timeout(time: 120, unit: 'MINUTES') { license: parts[0] == 'oss' ? 'oss' : 'default', ] } - .filter { !!it } + .findAll { !!it } sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' From 299815c33f2e356f56ca150553c838428d46f095 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 14:29:44 -0500 Subject: [PATCH 25/27] Some more cleanup and a better comment --- .ci/es-snapshots/Jenkinsfile_build_es | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index fbed42541a9372..67a2b6579cda08 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -50,11 +50,6 @@ timeout(time: 120, unit: 'MINUTES') { // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz def parts = filename.replace("elasticsearch-oss", "oss").split("-") - // We don't care about the no-jdk artifact - if (parts.size() < 5) { - return null - } - VERSION = VERSION ?: parts[1] SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" @@ -69,7 +64,6 @@ timeout(time: 120, unit: 'MINUTES') { license: parts[0] == 'oss' ? 'oss' : 'default', ] } - .findAll { !!it } sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' @@ -153,7 +147,7 @@ def buildArchives(destination) { // Select the correct JDK for this branch "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", - // These Jenkins env vars trigger some automation in ES that we don't want + // These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want "BUILD_NUMBER=", "JENKINS_URL=", "BUILD_URL=", @@ -163,7 +157,7 @@ def buildArchives(destination) { sh """ ./gradlew -p distribution/archives assemble --parallel mkdir -p ${destination} - find distribution/archives -type f \\( -name 'elasticsearch-*-*.tar.gz' -o -name 'elasticsearch-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\; + find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\; """ } } From d07885d8031ed21e69dbb2cde2802c959321b793 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 6 Jan 2020 14:24:19 -0500 Subject: [PATCH 26/27] Remove some TODOs --- .ci/es-snapshots/Jenkinsfile_build_es | 1 - vars/kibanaPipeline.groovy | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index 67a2b6579cda08..ad0ad54275e12d 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -67,7 +67,6 @@ timeout(time: 120, unit: 'MINUTES') { sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' - // TODO remove all currently unused params? def manifest = [ bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(), branch: ES_BRANCH, diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index c2da4acea715cd..c778dd799f6e59 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -237,8 +237,7 @@ def sendKibanaMail() { def buildStatus = buildUtils.getBuildStatus() if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED') { emailext( - // to: 'build-kibana@elastic.co', - to: 'brian.seeders@elastic.co', // TODO change this back + to: 'build-kibana@elastic.co', subject: "${env.JOB_NAME} - Build # ${env.BUILD_NUMBER} - ${buildStatus}", body: '${SCRIPT,template="groovy-html.template"}', mimeType: 'text/html', From 6b7257bc5a784c59b1d3c8745e330769ff37e348 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 6 Jan 2020 14:40:48 -0500 Subject: [PATCH 27/27] Fix snapshot unit test when ES_SNAPSHOT_MANIFEST is present before tests run --- packages/kbn-es/src/artifact.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/kbn-es/src/artifact.test.js b/packages/kbn-es/src/artifact.test.js index bff64da3ca4925..985b65c7475634 100644 --- a/packages/kbn-es/src/artifact.test.js +++ b/packages/kbn-es/src/artifact.test.js @@ -52,6 +52,23 @@ const createArchive = (params = {}) => { const mockFetch = mock => fetch.mockReturnValue(Promise.resolve(new Response(JSON.stringify(mock)))); +let previousSnapshotManifestValue = null; + +beforeAll(() => { + if ('ES_SNAPSHOT_MANIFEST' in process.env) { + previousSnapshotManifestValue = process.env.ES_SNAPSHOT_MANIFEST; + delete process.env.ES_SNAPSHOT_MANIFEST; + } +}); + +afterAll(() => { + if (previousSnapshotManifestValue !== null) { + process.env.ES_SNAPSHOT_MANIFEST = previousSnapshotManifestValue; + } else { + delete process.env.ES_SNAPSHOT_MANIFEST; + } +}); + beforeEach(() => { jest.resetAllMocks();