From ac465eba5d0706b57adec80a0202567a3bf22dcc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Jul 2020 17:22:08 +0100 Subject: [PATCH 001/118] [CI] Pipeline 2.0 for monorepos --- Jenkinsfile.yml | 29 +++++++++++++++++++++++++++ auditbeat/Jenkinsfile.yml | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 Jenkinsfile.yml create mode 100644 auditbeat/Jenkinsfile.yml diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml new file mode 100644 index 000000000000..fc602f44219c --- /dev/null +++ b/Jenkinsfile.yml @@ -0,0 +1,29 @@ +projects: + - "auditbeat" + - "filebeat" + - "heartbeat" + - "journalbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/journalbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" + +## Changeset macros that are defined here and used in each specific 2.0 pipeline. +changeset: + oss: + - "^Jenkinsfile" + - "^go.mod" + - "^libbeat/.*" + - "^testing/.*" + - "^dev-tools/.*" + - "^\\.ci/scripts/.*" diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..6dcd26c04eec --- /dev/null +++ b/auditbeat/Jenkinsfile.yml @@ -0,0 +1,42 @@ +when: + branches: true ## for all the branches + tags: true ## for all the tags + changeset: ## when PR contains any of those entries in the changeset + - "auditbeat" + - @oss' ## special token regarding the changeset for the oss + comments: + - "/test auditbeat" + labels: + - "auditbeat" + parameters: + - "auditbeat" +platform: ## default platform where the stages will use + - "linux && ubuntu-16" +stages: + build: + command: + - "cd auditbeat && mage build test" + crosscompile: + command: + - "make -C auditbeat crosscompile" + macos: + command: + - "cd auditbeat && mage build unitTest" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for macos" + labels: + - "macos" + parameters: + - "macos" + windows: + command: + - "cd auditbeat && mage build unitTest" + platform: + - "windows-2019" + - "windows-2016" + when: + comments: + - "/test auditbeat for windows" + parameters: + - "macos" From aa6a07aa3f43093c93424515289ddbe2c4470a60 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 23 Jul 2020 16:38:04 +0100 Subject: [PATCH 002/118] Update auditbeat/Jenkinsfile.yml Co-authored-by: Ivan Fernandez Calvo --- auditbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 6dcd26c04eec..4356a356d831 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -3,7 +3,7 @@ when: tags: true ## for all the tags changeset: ## when PR contains any of those entries in the changeset - "auditbeat" - - @oss' ## special token regarding the changeset for the oss + - '@oss' ## special token regarding the changeset for the oss comments: - "/test auditbeat" labels: From 1b91c12c0a9116d28b1f10f03fed0730c8c53409 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 23 Jul 2020 16:38:22 +0100 Subject: [PATCH 003/118] Update auditbeat/Jenkinsfile.yml Co-authored-by: Ivan Fernandez Calvo --- auditbeat/Jenkinsfile.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 4356a356d831..f627dfe84761 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -22,6 +22,8 @@ stages: macos: command: - "cd auditbeat && mage build unitTest" + platform: + - "macos" when: ## Aggregate when with the top-level one. comments: - "/test auditbeat for macos" From 7183f93dba35518d8a6f9a346d2c38798588b907 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 27 Jul 2020 12:49:05 +0100 Subject: [PATCH 004/118] [CI] 2.0 YAML to dynamic stage generations --- .ci/Jenkinsfile | 202 +++++++++++++++++++++++++++++++++++++ .ci/jobs/beats-mbp-2.0.yml | 60 +++++++++++ 2 files changed, 262 insertions(+) create mode 100644 .ci/Jenkinsfile create mode 100644 .ci/jobs/beats-mbp-2.0.yml diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile new file mode 100644 index 000000000000..f9065a61bba2 --- /dev/null +++ b/.ci/Jenkinsfile @@ -0,0 +1,202 @@ +#!/usr/bin/env groovy + +@Library('apm@current') _ + +pipeline { + agent { label 'ubuntu-18 && immutable' } + environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' + BASE_DIR = 'src/github.com/elastic/beats' + DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" + DOCKER_REGISTRY = 'docker.elastic.co' + GOX_FLAGS = "-arch amd64" + JOB_GCS_BUCKET = 'beats-ci-temp' + JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' + PIPELINE_LOG_LEVEL = 'INFO' + OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + } + options { + timeout(time: 2, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + quietPeriod(10) + rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) + } + triggers { + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') + } + parameters { + booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') + } + stages { + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) + deleteDir() + gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + script { + dir("${BASE_DIR}"){ + // Skip all the stages except docs for PR's with asciidoc and md changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) + } + } + } + } + stage('Lint'){ + options { skipDefaultCheckout() } + environment { + GOFLAGS = '-mod=readonly' + } + when { + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() + } + expression { return params.runAllStages } // If UI forced + } + } + steps { + echo 'linting' + } + } + stage('Build&Test') { + options { skipDefaultCheckout() } + steps { + deleteDir() + unstash "source" + dir("${BASE_DIR}"){ + script { + def masterFile = 'Jenkinsfile.yml' + def changeset = readYaml(file: masterFile)['changeset'] + readYaml(file: masterFile)['projects'].each { project -> + generateSteps(name: project, changeset: changeset).each { k,v -> + mapParallelTasks["${k}"] = v + } + } + parallel(mapParallelTasks) + } + } + } + } + } + post { + // TODO: Support runbld + cleanup { + notifyBuildResult(prComment: true) + } + } +} + +def generateSteps(Map args = [:]) { + def name = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def masterFile = "${name}/Jenkinsfile.yml" + def content = readYaml(file: masterFile) + def defaultNode = content['platform'] + + if (whenValidation(project: name, content: content, changeset: changeset)) { + content['stages'].each { stageName -> + if (stageName['when']) { + if (whenValidation(project: "stageName", content: stageName['when'], changeset: changeset)) { + generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> + mapParallelStages["${k}"] = v + } + } + } else { + generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> + mapParallelStages["${k}"] = v + } + } + } + return mapParallelStages +} + +def generateStages(Map args = [:]) { + def defaultNode = args.defaultNode + def mapParallelStages = [:] + if (content['platform']) { + content['platform']?.each { + mapParallelStages["${k}-${it}"] = generateStage(platform: it, content: stageName) + } + } else { + mapParallelStages["${k}"] = generateStage(platform: defaultNode, content: stageName) + } + return mapParallelStages +} + +def generateStage(Map args = [:]) { + return { + node(it) { + sh "${content['command']}" + } + } +} + +/** +* Given the YAML definition and the changeset global macros +* then it verifies if the project or stage should be enabled. +* +* It will generate a +*/ +def whenValidation(Map args = [:]) { + def project = args.project + def content = args.content + def patterns = args.changeset + def ret = false + if (content['comments']?.trim() && env.GITHUB_COMMENT?.trim()) { + if (content['comments']?.any { env.GITHUB_COMMENT?.toLowerCase()?.contains(it?.toLowerCase()) }) { + ret = true + markdownReason(project: project, reason: 'Comment is enabled and matches with the pattern.') + } else { + markdownReason(project: project, reason: 'Comment is enabled and does not match with the pattern.') + } + } + if (content['labels']) { + if (content['labels']?.any { matchesPrLabel(label: it) }) { + ret = true + markdownReason(project: project, reason: 'Label is enabled and matches with the pattern.') + } else { + markdownReason(project: project, reason: 'Label is enabled and does not match with the pattern.') + } + } + if (content['parameters']) { + if (content['parameters']?.any { params.[it] }) { + ret = true + markdownReason(project: project, reason: 'Parameter is enabled and matches with the pattern.') + } else { + markdownReason(project: project, reason: 'Parameter is enabled and does not match with the pattern.') + } + } + if (content['branches'] && env.BRANCH_NAME?.trim()) { + ret = true + markdownReason(project: project, reason: 'Branch is enabled and matches with the pattern.') + } + if (content['tags'] && env.TAG_NAME?.trim()) { + ret = true + markdownReason(project: project, reason: 'Tag is enabled and matches with the pattern.') + } + return ret +} + +def markdownReason(Map args = [:]) { + echo "${args.project} - ${args.reason}" + // TODO create markdown +} \ No newline at end of file diff --git a/.ci/jobs/beats-mbp-2.0.yml b/.ci/jobs/beats-mbp-2.0.yml new file mode 100644 index 000000000000..d1057dd3047a --- /dev/null +++ b/.ci/jobs/beats-mbp-2.0.yml @@ -0,0 +1,60 @@ +--- +- job: + name: Beats/beats-mbp-2.0 + display-name: 'Beats (2.0)' + description: 'Beats Main Pipeline 2.0' + view: Beats + concurrent: true + project-type: multibranch + prune-dead-branches: true + days-to-keep: 30 + script-path: '.ci/Jenkinsfile' + triggers: [] + wrappers: [] + scm: + - github: + branch-discovery: 'no-pr' + discover-pr-forks-strategy: 'merge-current' + discover-pr-forks-trust: 'permission' + discover-pr-origin: 'merge-current' + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' + discover-tags: true + disable-pr-notifications: true + notification-context: "beats-ci-2.0" + repo: 'beats' + repo-owner: 'elastic' + credentials-id: github-app-beats-ci + ssh-checkout: + credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba + build-strategies: + - skip-initial-build: true + - tags: + ignore-tags-older-than: -1 + ignore-tags-newer-than: 365 + - change-request: + ignore-target-only-changes: true + - named-branches: + - exact-name: + name: 'master' + case-sensitive: true + - regex-name: + regex: '7\.[x789]' + case-sensitive: true + - regex-name: + regex: '8\.\d+' + case-sensitive: true + clean: + after: true + before: true + prune: true + shallow-clone: true + depth: 3 + do-not-fetch-tags: true + submodule: + disable: false + recursive: true + parent-credentials: true + timeout: 100 + timeout: '15' + use-author: true + wipe-workspace: true From ca9a5a6b5708d9f94c0b3ee7f932f1b1d5dd2fff Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 27 Jul 2020 17:26:28 +0100 Subject: [PATCH 005/118] Use shared library --- .ci/Jenkinsfile | 57 +++---------------------------------------------- 1 file changed, 3 insertions(+), 54 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index f9065a61bba2..2a6ed9c7c6f0 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -1,6 +1,6 @@ #!/usr/bin/env groovy -@Library('apm@current') _ +@Library('apm@feature/2.0-beats') _ pipeline { agent { label 'ubuntu-18 && immutable' } @@ -112,10 +112,10 @@ def generateSteps(Map args = [:]) { def content = readYaml(file: masterFile) def defaultNode = content['platform'] - if (whenValidation(project: name, content: content, changeset: changeset)) { + if (beatsWhen(project: name, content: content, changeset: changeset)) { content['stages'].each { stageName -> if (stageName['when']) { - if (whenValidation(project: "stageName", content: stageName['when'], changeset: changeset)) { + if (beatsWhen(project: "stageName", content: stageName['when'], changeset: changeset)) { generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> mapParallelStages["${k}"] = v } @@ -148,55 +148,4 @@ def generateStage(Map args = [:]) { sh "${content['command']}" } } -} - -/** -* Given the YAML definition and the changeset global macros -* then it verifies if the project or stage should be enabled. -* -* It will generate a -*/ -def whenValidation(Map args = [:]) { - def project = args.project - def content = args.content - def patterns = args.changeset - def ret = false - if (content['comments']?.trim() && env.GITHUB_COMMENT?.trim()) { - if (content['comments']?.any { env.GITHUB_COMMENT?.toLowerCase()?.contains(it?.toLowerCase()) }) { - ret = true - markdownReason(project: project, reason: 'Comment is enabled and matches with the pattern.') - } else { - markdownReason(project: project, reason: 'Comment is enabled and does not match with the pattern.') - } - } - if (content['labels']) { - if (content['labels']?.any { matchesPrLabel(label: it) }) { - ret = true - markdownReason(project: project, reason: 'Label is enabled and matches with the pattern.') - } else { - markdownReason(project: project, reason: 'Label is enabled and does not match with the pattern.') - } - } - if (content['parameters']) { - if (content['parameters']?.any { params.[it] }) { - ret = true - markdownReason(project: project, reason: 'Parameter is enabled and matches with the pattern.') - } else { - markdownReason(project: project, reason: 'Parameter is enabled and does not match with the pattern.') - } - } - if (content['branches'] && env.BRANCH_NAME?.trim()) { - ret = true - markdownReason(project: project, reason: 'Branch is enabled and matches with the pattern.') - } - if (content['tags'] && env.TAG_NAME?.trim()) { - ret = true - markdownReason(project: project, reason: 'Tag is enabled and matches with the pattern.') - } - return ret -} - -def markdownReason(Map args = [:]) { - echo "${args.project} - ${args.reason}" - // TODO create markdown } \ No newline at end of file From 56c45d51774e13f161da75bf99bb9c37a4bc245f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 14:20:11 +0100 Subject: [PATCH 006/118] Use beatsWhen and beatsStages --- .ci/Jenkinsfile | 151 ---- Jenkinsfile | 1426 ++----------------------------------- auditbeat/Jenkinsfile.yml | 25 +- 3 files changed, 58 insertions(+), 1544 deletions(-) delete mode 100644 .ci/Jenkinsfile diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index 2a6ed9c7c6f0..000000000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@feature/2.0-beats') _ - -pipeline { - agent { label 'ubuntu-18 && immutable' } - environment { - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - BASE_DIR = 'src/github.com/elastic/beats' - DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_COMPOSE_VERSION = "1.21.0" - DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" - JOB_GCS_BUCKET = 'beats-ci-temp' - JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - PIPELINE_LOG_LEVEL = 'INFO' - OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' - TERRAFORM_VERSION = "0.12.24" - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - } - options { - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - quietPeriod(10) - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - } - triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') - } - parameters { - booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') - string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - script { - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc and md changes only - env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) - } - } - } - } - stage('Lint'){ - options { skipDefaultCheckout() } - environment { - GOFLAGS = '-mod=readonly' - } - when { - anyOf { - not { changeRequest() } // If no PR - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - echo 'linting' - } - } - stage('Build&Test') { - options { skipDefaultCheckout() } - steps { - deleteDir() - unstash "source" - dir("${BASE_DIR}"){ - script { - def masterFile = 'Jenkinsfile.yml' - def changeset = readYaml(file: masterFile)['changeset'] - readYaml(file: masterFile)['projects'].each { project -> - generateSteps(name: project, changeset: changeset).each { k,v -> - mapParallelTasks["${k}"] = v - } - } - parallel(mapParallelTasks) - } - } - } - } - } - post { - // TODO: Support runbld - cleanup { - notifyBuildResult(prComment: true) - } - } -} - -def generateSteps(Map args = [:]) { - def name = args.project - def changeset = args.changeset - def mapParallelStages = [:] - def masterFile = "${name}/Jenkinsfile.yml" - def content = readYaml(file: masterFile) - def defaultNode = content['platform'] - - if (beatsWhen(project: name, content: content, changeset: changeset)) { - content['stages'].each { stageName -> - if (stageName['when']) { - if (beatsWhen(project: "stageName", content: stageName['when'], changeset: changeset)) { - generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> - mapParallelStages["${k}"] = v - } - } - } else { - generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> - mapParallelStages["${k}"] = v - } - } - } - return mapParallelStages -} - -def generateStages(Map args = [:]) { - def defaultNode = args.defaultNode - def mapParallelStages = [:] - if (content['platform']) { - content['platform']?.each { - mapParallelStages["${k}-${it}"] = generateStage(platform: it, content: stageName) - } - } else { - mapParallelStages["${k}"] = generateStage(platform: defaultNode, content: stageName) - } - return mapParallelStages -} - -def generateStage(Map args = [:]) { - return { - node(it) { - sh "${content['command']}" - } - } -} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index c415e7ef6057..be980f97bbbf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,43 +1,23 @@ #!/usr/bin/env groovy -@Library('apm@current') _ - -import groovy.transform.Field - - -/** - NOTE: Important note regarding the agents and labels. - agent labels are defined in the gobld service, that's managed by infra. The required format - is: - - ' && immutable' for linux OS. - - 'macosx' for the MacOS. - - 'windows-immutable && windows-' for Windows. NOTE: version might differ in some cases - - The above labels will help to set what OS family and specific version of the agent is - required to used in the stage. -*/ - -/** - This is required to store the stashed id with the test results to be digested with runbld -*/ -@Field def stashedTestReports = [:] +@Library('apm@feature/2.0-beats') _ pipeline { agent { label 'ubuntu-18 && immutable' } environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' BASE_DIR = 'src/github.com/elastic/beats' - GOX_FLAGS = "-arch amd64" - DOCKER_COMPOSE_VERSION = "1.21.0" - TERRAFORM_VERSION = "0.12.24" - PIPELINE_LOG_LEVEL = "INFO" DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' + GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + PIPELINE_LOG_LEVEL = 'INFO' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { timeout(time: 2, unit: 'HOURS') @@ -50,22 +30,18 @@ pipeline { rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) } triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test(\\W+macos)?$)') + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') } parameters { - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') - booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') } stages { - /** - Checkout the code and stash it, to use it on other stages. - */ stage('Checkout') { options { skipDefaultCheckout() } steps { @@ -73,1372 +49,62 @@ pipeline { deleteDir() gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - loadConfigEnvVars() - } - whenTrue(params.debug){ - dumpFilteredEnvironment() + script { + dir("${BASE_DIR}"){ + // Skip all the stages except docs for PR's with asciidoc and md changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) + } } } } stage('Lint'){ options { skipDefaultCheckout() } environment { - // See https://github.com/elastic/beats/pull/19823 GOFLAGS = '-mod=readonly' } - steps { - makeTarget(context: "Lint", target: "check") - } - } - stage('Build and Test'){ when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - parallel { - stage('Elastic Agent x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Linux", directory: "x-pack/elastic-agent", target: "build test") - } - } - stage('Elastic Agent x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Elastic Agent x-pack Windows Unit test", directory: "x-pack/elastic-agent", target: "build unitTest") - } - } - stage('Elastic Agent Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Mac OS X", directory: "x-pack/elastic-agent", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" - } - } - steps { - mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat oss Mac OS X", directory: "filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat x-pack Mac OS X", directory: "x-pack/filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat oss Windows Unit test", directory: "filebeat", target: "build unitTest") - } - } - stage('Filebeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat x-pack Windows", directory: "x-pack/filebeat", target: "build unitTest") - } - } - stage('Heartbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_HEARTBEAT != "false" - } - } - stages { - stage('Heartbeat oss'){ - steps { - mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") - } - } - stage('Heartbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Heartbeat oss Mac OS X", directory: "heartbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Heartbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin(context: "Heartbeat oss Windows Unit test", directory: "heartbeat", target: "build unitTest") - } - } - } - } - stage('Auditbeat oss Linux'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") - } - } - stage('Auditbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile") - } - } - stage('Auditbeat oss Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat oss Mac OS X", directory: "auditbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Auditbeat oss Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat oss Windows Unit test", directory: "auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Auditbeat x-pack Linux", directory: "x-pack/auditbeat", target: "update build test", withModule: true) - } - } - stage('Auditbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat x-pack Mac OS X", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat x-pack Windows", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Libbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT != "false" - } - } - stages { - stage('Libbeat oss'){ - steps { - mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") - } - } - stage('Libbeat crosscompile'){ - steps { - makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile") - } - } - stage('Libbeat stress-tests'){ - steps { - makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") - } - } - } - } - stage('Libbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") - } - } - stage('Metricbeat OSS Unit tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat OSS Go Integration tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) - } - } - stage('Metricbeat OSS Python Integration tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (pythonIntegTest)", directory: "metricbeat", target: "pythonIntegTest", withModule: true) - } - } - stage('Metricbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" - } - } - stages { - stage('Prepare cloud integration tests environments'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - steps { - startCloudTestEnv('x-pack-metricbeat', [ - [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], - ]) - } - } - stage('Metricbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - steps { - withCloudTestEnv() { - mageTarget(context: "Metricbeat x-pack Linux", directory: "x-pack/metricbeat", target: "build test", withModule: true) - } - } - } - } - post { - cleanup { - terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') - } - } - } - stage('Metricbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile") - } - } - stage('Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat OSS Mac OS X", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat x-pack Mac OS X", directory: "x-pack/metricbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Metricbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat Windows Unit test", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") - } - } - stage('Packetbeat OSS'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_PACKETBEAT != "false" - } - } - stages { - stage('Packetbeat Linux'){ - steps { - mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") - } - } - stage('Packetbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Packetbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") - } - } - } - } - stage('dockerlogbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_DOCKERLOGBEAT_XPACK != "false" - } - } - stages { - stage('Dockerlogbeat'){ - steps { - mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") - } - } - } - } - stage('Winlogbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_WINLOGBEAT != "false" - } - } - stages { - stage('Winlogbeat oss'){ - steps { - makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile") - } - } - stage('Winlogbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "winlogbeat", target: "build unitTest") - } - } - } - } - stage('Winlogbeat Windows x-pack'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "x-pack/winlogbeat", target: "build unitTest", withModule: true) - } - } - stage('Functionbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - stages { - stage('Functionbeat x-pack'){ - steps { - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") - withEnv(["GO_VERSION=1.13.1"]){ - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") - } - } - } - stage('Functionbeat Mac OS X x-pack'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Functionbeat x-pack Mac OS X", directory: "x-pack/functionbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Functionbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin(context: "Functionbeat Windows Unit test", directory: "x-pack/functionbeat", target: "build unitTest") - } - } - } - } - stage('Journalbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_JOURNALBEAT != "false" - } - } - stages { - stage('Journalbeat oss'){ - steps { - mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build unitTest") - } - } - } - } - stage('Generators'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_GENERATOR != "false" - } - } - stages { - stage('Generators Metricbeat Linux'){ - steps { - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test") - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package") - } - } - stage('Generators Beat Linux'){ - steps { - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test") - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package") - } - } - stage('Generators Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' - } - } - steps { - makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test") - } - post { - always { - delete() - } - } - } - stage('Generators Beat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' - } - } - steps { - makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test") - } - post { - always { - delete() - } - } - } - } - } - stage('Kubernetes'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_KUBERNETES != "false" - } - } - steps { - k8sTest(["v1.18.2","v1.17.2","v1.16.4","v1.15.7","v1.14.10"]) + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() } + expression { return params.runAllStages } // If UI forced } } - } - } - post { - always { - runbld() - } - cleanup { - notifyBuildResult(prComment: true) - } - } -} - -def delete() { - dir("${env.BASE_DIR}") { - fixPermissions("${WORKSPACE}") - } - deleteDir() -} - -def fixPermissions(location) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) -} - -def makeTarget(Map args = [:]) { - def context = args.context - def target = args.target - def directory = args.get('directory', '') - def clean = args.get('clean', true) - def withModule = args.get('withModule', false) - def directoryFlag = directory.trim() ? "-C ${directory}" : '' - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") - whenTrue(clean) { - fixPermissions("${HOME}") - } - } - } -} - -def mageTarget(Map args = [:]) { - def context = args.context - def directory = args.directory - def target = args.target - def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") - } - } - } -} - -def mageTargetWin(Map args = [:]) { - def context = args.context - def directory = args.directory - def target = args.target - def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnvWin(withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMageWin() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") - } - } - } -} - -def getModulePattern(String toCompare) { - // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') - return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) -} - -def withBeatsEnv(Map args = [:], Closure body) { - def archive = args.get('archive', true) - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) - } - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - - withEnv([ - "HOME=${env.WORKSPACE}", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - "MAGEFILE_CACHE=${WORKSPACE}/.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - "PYTHON_ENV=${WORKSPACE}/python-env", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "DOCKER_PULL=0", - "MODULE=${module}" - ]) { - if(isDockerInstalled()){ - dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - dir("${env.BASE_DIR}") { - installTools() - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - setGitConfig() - try { - if(!params.dry_run){ - body() - } - } finally { - if (archive) { - archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out') - } - } - } - } -} - -/** - This method archives and report the tests output, for such, it searches in certain folders - to bypass some issues when working with big repositories. -*/ -def archiveTestOutput(Map args = [:]) { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - if (isUnix()) { - fixPermissions("${WORKSPACE}") - } - cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') - dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) - archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) - } - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") - if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() - tar(file: "${name}.tgz", archive: true, dir: folder) - } - } - } -} - -def withBeatsEnvWin(Map args = [:], Closure body) { - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) - } - final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" - - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - - withEnv([ - "HOME=${env.WORKSPACE}", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", - "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - "MODULE=${module}" - ]){ - dir("${env.BASE_DIR}"){ - installTools() - try { - if(!params.dry_run){ - body() - } - } finally { - archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out") - } - } - } -} - -def installTools() { - def i = 2 // Number of retries - if(isUnix()) { - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") } - } else { - retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } - } -} - -def goos(){ - def labels = env.NODE_LABELS - - if (labels.contains('linux')) { - return 'linux' - } else if (labels.contains('windows')) { - return 'windows' - } else if (labels.contains('darwin')) { - return 'darwin' - } - - error("Unhandled OS name in NODE_LABELS: " + labels) -} - -def dumpMage(){ - echo "### MAGE DUMP ###" - sh(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpMageWin(){ - echo "### MAGE DUMP ###" - bat(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpFilteredEnvironment(){ - echo "### ENV DUMP ###" - echo "PATH: ${env.PATH}" - echo "HOME: ${env.HOME}" - echo "USERPROFILE: ${env.USERPROFILE}" - echo "BUILD_DIR: ${env.BUILD_DIR}" - echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" - echo "BEATS: ${env.BEATS}" - echo "PROJECTS: ${env.PROJECTS}" - echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" - echo "PYTHON_ENV: ${env.PYTHON_ENV}" - echo "PYTHON_EXE: ${env.PYTHON_EXE}" - echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" - echo "VENV_PARAMS: ${env.VENV_PARAMS}" - echo "FIND: ${env.FIND}" - echo "GOLINT: ${env.GOLINT}" - echo "GOLINT_REPO: ${env.GOLINT_REPO}" - echo "REVIEWDOG: ${env.REVIEWDOG}" - echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" - echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" - echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" - echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" - echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" - echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" - echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" - echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" - echo "NOW: ${env.NOW}" - echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" - echo "GOIMPORTS: ${env.GOIMPORTS}" - echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" - echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" - echo "PROCESSES: ${env.PROCESSES}" - echo "TIMEOUT: ${env.TIMEOUT}" - echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" - echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" - echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" - echo "STRESS_TESTS: ${env.STRESS_TESTS}" - echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" - echo "TEST_TAGS: ${env.TEST_TAGS}" - echo "GOX_OS: ${env.GOX_OS}" - echo "GOX_OSARCH: ${env.GOX_OSARCH}" - echo "GOX_FLAGS: ${env.GOX_FLAGS}" - echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" - echo "BEAT_VERSION: ${env.BEAT_VERSION}" - echo "COMMIT_ID: ${env.COMMIT_ID}" - echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" - echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" - echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" - echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" - echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" - echo "### END ENV DUMP ###" -} - -def k8sTest(versions){ - versions.each{ v -> - stage("k8s ${v}"){ - withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(archive: false, withModule: false) { - sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") - sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - sh(label: 'Delete cluster', script: 'kind delete cluster') - } - } + steps { + echo 'linting' } } - } -} - -/** -* isChanged treats the patterns as regular expressions. In order to check if -* any file in a directoy is modified use `^/.*`. -* -* In addition, there are another two alternatives to report that there are -* changes, when `runAllStages` parameter is set to true or when running on a -* branch/tag basis. -*/ -def isChanged(patterns){ - return ( - params.runAllStages // when runAllStages UI parameter is set to true - || !isPR() // when running on a branch/tag - || isGitRegionMatch(patterns: patterns, comparator: 'regexp') - ) -} - -def isChangedOSSCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^libbeat/.*", - "^testing/.*", - "^dev-tools/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -def isChangedXPackCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^libbeat/.*", - "^dev-tools/.*", - "^testing/.*", - "^x-pack/libbeat/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -// withCloudTestEnv executes a closure with credentials for cloud test -// environments. -def withCloudTestEnv(Closure body) { - def maskedVars = [] - def testTags = "${env.TEST_TAGS}" - - // AWS - if (params.allCloudTests || params.awsCloudTests) { - testTags = "${testTags},aws" - def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data - if (!aws.containsKey('access_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") - } - if (!aws.containsKey('secret_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") - } - maskedVars.addAll([ - [var: "AWS_REGION", password: params.awsRegion], - [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], - [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], - ]) - } - - withEnv([ - "TEST_TAGS=${testTags}", - ]) { - withEnvMask(vars: maskedVars) { - body() - } - } -} - -def terraformInit(String directory) { - dir(directory) { - sh(label: "Terraform Init on ${directory}", script: "terraform init") - } -} - -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } -} - -// Start testing environment on cloud using terraform. Terraform files are -// stashed so they can be used by other stages. They are also archived in -// case manual cleanup is needed. -// -// Example: -// startCloudTestEnv('x-pack-metricbeat', [ -// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], -// ]) -// ... -// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') -def startCloudTestEnv(String name, environments = []) { - withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { - def runAll = params.runAllCloudTests - try { - for (environment in environments) { - if (environment.cond || runAll) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(environment.dir) + stage('Build&Test') { + options { skipDefaultCheckout() } + steps { + deleteDir() + unstash "source" + dir("${BASE_DIR}"){ + script { + def masterFile = 'Jenkinsfile.yml' + def changeset = readYaml(file: masterFile)['changeset'] + readYaml(file: masterFile)['projects'].each { projectName -> + generateStages(project: projectName, changeset: changeset).each { k,v -> + mapParallelTasks["${k}"] = v + } } + parallel(mapParallelTasks) } } - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') - } - } -} - - -// Looks for all terraform states in directory and runs terraform destroy for them, -// it uses terraform states previously stashed by startCloudTestEnv. -def terraformCleanup(String stashName, String directory) { - stage("Remove cloud scenarios in ${directory}"){ - withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${stashName}") - retryWithSleep(retries: 2, seconds: 5, backoff: true) { - sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") + post { + always { + echo 'TODO: save markdown files for the record' } } } } -} - -def loadConfigEnvVars(){ - def empty = [] - env.GO_VERSION = readFile(".go-version").trim() - - withEnv(["HOME=${env.WORKSPACE}"]) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") } - } - - // Libbeat is the core framework of Beats. It has no additional dependencies - // on other projects in the Beats repository. - env.BUILD_LIBBEAT = isChangedOSSCode(empty) - env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) - - // Auditbeat depends on metricbeat as framework, but does not include any of - // the modules from Metricbeat. - // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) - - // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) - - // Filebeat depends on libbeat only. - // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) - - // Metricbeat depends on libbeat only. - // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) - - // Functionbeat is a standalone beat that depends on libbeat only. - // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) - - // Heartbeat depends on libbeat only. - // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) - - // Journalbeat depends on libbeat only. - // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) - - // Packetbeat depends on libbeat only. - // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) - - // Winlogbeat depends on libbeat only. - // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) - - // Elastic-agent is a self-contained product, that depends on libbeat only. - // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. - // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) - - // The Kubernetes test use Filebeat and Metricbeat, but only need to be run - // if the deployment scripts have been updated. No Beats specific testing is - // involved. - env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - - def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) - generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) - env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) - - // Skip all the stages for changes only related to the documentation - env.ONLY_DOCS = isDocChangedOnly() - - // Enable macOS builds when required - env.BUILD_ON_MACOS = (params.macosTest // UI Input parameter is set to true - || !isPR() // For branches and tags - || matchesPrLabel(label: 'macOS') // If `macOS` GH label (Case-Sensitive) - || (env.GITHUB_COMMENT?.toLowerCase()?.contains('/test macos'))) // If `/test macos` in the GH comment (Case-Insensitive) -} - -/** - This method gathers the module name, if required, in order to run the ITs only if - the changeset affects a specific module. - - For such, it's required to look for changes under the module folder and exclude anything else - such as ascidoc and png files. -*/ -def getCommonModuleInTheChangeSet(String pattern, String directory) { - def module = '' - // Transform folder structure in regex format since path separator is required to be escaped - def transformedDirectory = directory.replaceAll('/', '\\/') - def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" - def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" - dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern, exclude: exclude) - } - return module -} - -/** - This method verifies if the changeset for the current pull request affect only changes related - to documentation, such as asciidoc and png files. -*/ -def isDocChangedOnly(){ - if (params.runAllStages || !env.CHANGE_ID?.trim()) { - log(level: 'INFO', text: 'Speed build for docs only is disabled for branches/tags or when forcing with the runAllStages parameter.') - return 'false' - } else { - log(level: "INFO", text: 'Check if the speed build for docs is enabled.') - return isGitRegionMatch(patterns: ['.*\\.(asciidoc|png)'], shouldMatchAll: true) - } -} - -/** - This method grab the dependencies of a Go module and transform them on regexp -*/ -def getProjectDependencies(beatName){ - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - def output = "" - - withEnv([ - "HOME=${env.WORKSPACE}/${env.BASE_DIR}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - ]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) - } - return output?.split('\n').collect{ item -> item as String } -} - -def setGitConfig(){ - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" - fi - ''') -} - -def isDockerInstalled(){ - return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) -} - -def junitAndStore(Map params = [:]){ - junit(params) - // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds - def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" - stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) - stashedTestReports[stageName] = stageName -} - -def runbld() { - catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { - if (stashedTestReports) { - dir("${env.BASE_DIR}") { - sh(label: 'Prepare workspace context', - script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete') - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } - } - sh(label: 'Process JUnit reports with runbld', - script: '''\ - cat >./runbld-script < Date: Tue, 28 Jul 2020 14:30:35 +0100 Subject: [PATCH 007/118] Revert "Use beatsWhen and beatsStages" This reverts commit 56c45d51774e13f161da75bf99bb9c37a4bc245f. --- .ci/Jenkinsfile | 151 ++++ Jenkinsfile | 1426 +++++++++++++++++++++++++++++++++++-- auditbeat/Jenkinsfile.yml | 25 +- 3 files changed, 1544 insertions(+), 58 deletions(-) create mode 100644 .ci/Jenkinsfile diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile new file mode 100644 index 000000000000..2a6ed9c7c6f0 --- /dev/null +++ b/.ci/Jenkinsfile @@ -0,0 +1,151 @@ +#!/usr/bin/env groovy + +@Library('apm@feature/2.0-beats') _ + +pipeline { + agent { label 'ubuntu-18 && immutable' } + environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' + BASE_DIR = 'src/github.com/elastic/beats' + DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" + DOCKER_REGISTRY = 'docker.elastic.co' + GOX_FLAGS = "-arch amd64" + JOB_GCS_BUCKET = 'beats-ci-temp' + JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' + PIPELINE_LOG_LEVEL = 'INFO' + OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + } + options { + timeout(time: 2, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + quietPeriod(10) + rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) + } + triggers { + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') + } + parameters { + booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') + } + stages { + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) + deleteDir() + gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + script { + dir("${BASE_DIR}"){ + // Skip all the stages except docs for PR's with asciidoc and md changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) + } + } + } + } + stage('Lint'){ + options { skipDefaultCheckout() } + environment { + GOFLAGS = '-mod=readonly' + } + when { + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() + } + expression { return params.runAllStages } // If UI forced + } + } + steps { + echo 'linting' + } + } + stage('Build&Test') { + options { skipDefaultCheckout() } + steps { + deleteDir() + unstash "source" + dir("${BASE_DIR}"){ + script { + def masterFile = 'Jenkinsfile.yml' + def changeset = readYaml(file: masterFile)['changeset'] + readYaml(file: masterFile)['projects'].each { project -> + generateSteps(name: project, changeset: changeset).each { k,v -> + mapParallelTasks["${k}"] = v + } + } + parallel(mapParallelTasks) + } + } + } + } + } + post { + // TODO: Support runbld + cleanup { + notifyBuildResult(prComment: true) + } + } +} + +def generateSteps(Map args = [:]) { + def name = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def masterFile = "${name}/Jenkinsfile.yml" + def content = readYaml(file: masterFile) + def defaultNode = content['platform'] + + if (beatsWhen(project: name, content: content, changeset: changeset)) { + content['stages'].each { stageName -> + if (stageName['when']) { + if (beatsWhen(project: "stageName", content: stageName['when'], changeset: changeset)) { + generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> + mapParallelStages["${k}"] = v + } + } + } else { + generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> + mapParallelStages["${k}"] = v + } + } + } + return mapParallelStages +} + +def generateStages(Map args = [:]) { + def defaultNode = args.defaultNode + def mapParallelStages = [:] + if (content['platform']) { + content['platform']?.each { + mapParallelStages["${k}-${it}"] = generateStage(platform: it, content: stageName) + } + } else { + mapParallelStages["${k}"] = generateStage(platform: defaultNode, content: stageName) + } + return mapParallelStages +} + +def generateStage(Map args = [:]) { + return { + node(it) { + sh "${content['command']}" + } + } +} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index be980f97bbbf..c415e7ef6057 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,23 +1,43 @@ #!/usr/bin/env groovy -@Library('apm@feature/2.0-beats') _ +@Library('apm@current') _ + +import groovy.transform.Field + + +/** + NOTE: Important note regarding the agents and labels. + agent labels are defined in the gobld service, that's managed by infra. The required format + is: + - ' && immutable' for linux OS. + - 'macosx' for the MacOS. + - 'windows-immutable && windows-' for Windows. NOTE: version might differ in some cases + + The above labels will help to set what OS family and specific version of the agent is + required to used in the stage. +*/ + +/** + This is required to store the stashed id with the test results to be digested with runbld +*/ +@Field def stashedTestReports = [:] pipeline { agent { label 'ubuntu-18 && immutable' } environment { - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' BASE_DIR = 'src/github.com/elastic/beats' - DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + GOX_FLAGS = "-arch amd64" DOCKER_COMPOSE_VERSION = "1.21.0" + TERRAFORM_VERSION = "0.12.24" + PIPELINE_LOG_LEVEL = "INFO" + DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - PIPELINE_LOG_LEVEL = 'INFO' - OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' - TERRAFORM_VERSION = "0.12.24" XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { timeout(time: 2, unit: 'HOURS') @@ -30,18 +50,22 @@ pipeline { rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) } triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test(\\W+macos)?$)') } parameters { + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') + booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') } stages { + /** + Checkout the code and stash it, to use it on other stages. + */ stage('Checkout') { options { skipDefaultCheckout() } steps { @@ -49,62 +73,1372 @@ pipeline { deleteDir() gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - script { - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc and md changes only - env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) - } + dir("${BASE_DIR}"){ + loadConfigEnvVars() + } + whenTrue(params.debug){ + dumpFilteredEnvironment() } } } stage('Lint'){ options { skipDefaultCheckout() } environment { + // See https://github.com/elastic/beats/pull/19823 GOFLAGS = '-mod=readonly' } - when { - anyOf { - not { changeRequest() } // If no PR - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } steps { - echo 'linting' + makeTarget(context: "Lint", target: "check") } } - stage('Build&Test') { - options { skipDefaultCheckout() } - steps { - deleteDir() - unstash "source" - dir("${BASE_DIR}"){ - script { - def masterFile = 'Jenkinsfile.yml' - def changeset = readYaml(file: masterFile)['changeset'] - readYaml(file: masterFile)['projects'].each { projectName -> - generateStages(project: projectName, changeset: changeset).each { k,v -> - mapParallelTasks["${k}"] = v + stage('Build and Test'){ + when { + beforeAgent true + expression { return env.ONLY_DOCS == "false" } + } + failFast false + parallel { + stage('Elastic Agent x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ELASTIC_AGENT_XPACK != "false" + } + } + steps { + mageTarget(context: "Elastic Agent x-pack Linux", directory: "x-pack/elastic-agent", target: "build test") + } + } + stage('Elastic Agent x-pack Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ELASTIC_AGENT_XPACK != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Elastic Agent x-pack Windows Unit test", directory: "x-pack/elastic-agent", target: "build unitTest") + } + } + stage('Elastic Agent Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ELASTIC_AGENT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Elastic Agent x-pack Mac OS X", directory: "x-pack/elastic-agent", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Filebeat oss'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" + } + } + steps { + mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) + } + } + stage('Filebeat x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" + } + } + steps { + mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) + } + } + stage('Filebeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Filebeat oss Mac OS X", directory: "filebeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Filebeat x-pack Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Filebeat x-pack Mac OS X", directory: "x-pack/filebeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Filebeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Filebeat oss Windows Unit test", directory: "filebeat", target: "build unitTest") + } + } + stage('Filebeat x-pack Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Filebeat x-pack Windows", directory: "x-pack/filebeat", target: "build unitTest") + } + } + stage('Heartbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_HEARTBEAT != "false" + } + } + stages { + stage('Heartbeat oss'){ + steps { + mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") + } + } + stage('Heartbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Heartbeat oss Mac OS X", directory: "heartbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Heartbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin(context: "Heartbeat oss Windows Unit test", directory: "heartbeat", target: "build unitTest") } } - parallel(mapParallelTasks) } } - } - post { - always { - echo 'TODO: save markdown files for the record' + stage('Auditbeat oss Linux'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" + } + } + steps { + mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") + } + } + stage('Auditbeat crosscompile'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" + } + } + steps { + makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile") + } + } + stage('Auditbeat oss Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Auditbeat oss Mac OS X", directory: "auditbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Auditbeat oss Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Auditbeat oss Windows Unit test", directory: "auditbeat", target: "build unitTest") + } + } + stage('Auditbeat x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" + } + } + steps { + mageTarget(context: "Auditbeat x-pack Linux", directory: "x-pack/auditbeat", target: "update build test", withModule: true) + } + } + stage('Auditbeat x-pack Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Auditbeat x-pack Mac OS X", directory: "x-pack/auditbeat", target: "build unitTest") + } + } + stage('Auditbeat x-pack Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Auditbeat x-pack Windows", directory: "x-pack/auditbeat", target: "build unitTest") + } + } + stage('Libbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT != "false" + } + } + stages { + stage('Libbeat oss'){ + steps { + mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") + } + } + stage('Libbeat crosscompile'){ + steps { + makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile") + } + } + stage('Libbeat stress-tests'){ + steps { + makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") + } + } + } + } + stage('Libbeat x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT_XPACK != "false" + } + } + steps { + mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") + } + } + stage('Metricbeat OSS Unit tests'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") + } + } + stage('Metricbeat OSS Go Integration tests'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) + } + } + stage('Metricbeat OSS Python Integration tests'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + mageTarget(context: "Metricbeat OSS linux/amd64 (pythonIntegTest)", directory: "metricbeat", target: "pythonIntegTest", withModule: true) + } + } + stage('Metricbeat x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT_XPACK != "false" + } + } + stages { + stage('Prepare cloud integration tests environments'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + steps { + startCloudTestEnv('x-pack-metricbeat', [ + [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], + ]) + } + } + stage('Metricbeat x-pack'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + steps { + withCloudTestEnv() { + mageTarget(context: "Metricbeat x-pack Linux", directory: "x-pack/metricbeat", target: "build test", withModule: true) + } + } + } + } + post { + cleanup { + terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') + } + } + } + stage('Metricbeat crosscompile'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile") + } + } + stage('Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Metricbeat OSS Mac OS X", directory: "metricbeat", target: "build unitTest") + } + } + stage('Metricbeat x-pack Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Metricbeat x-pack Mac OS X", directory: "x-pack/metricbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Metricbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Metricbeat Windows Unit test", directory: "metricbeat", target: "build unitTest") + } + } + stage('Metricbeat x-pack Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT_XPACK != "false" && params.windowsTest + } + } + steps { + mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") + } + } + stage('Packetbeat OSS'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_PACKETBEAT != "false" + } + } + stages { + stage('Packetbeat Linux'){ + steps { + mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") + } + } + stage('Packetbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Packetbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") + } + } + } + } + stage('dockerlogbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_DOCKERLOGBEAT_XPACK != "false" + } + } + stages { + stage('Dockerlogbeat'){ + steps { + mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") + } + } + } + } + stage('Winlogbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_WINLOGBEAT != "false" + } + } + stages { + stage('Winlogbeat oss'){ + steps { + makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile") + } + } + stage('Winlogbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "winlogbeat", target: "build unitTest") + } + } + } + } + stage('Winlogbeat Windows x-pack'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" + } + } + steps { + mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "x-pack/winlogbeat", target: "build unitTest", withModule: true) + } + } + stage('Functionbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FUNCTIONBEAT_XPACK != "false" + } + } + stages { + stage('Functionbeat x-pack'){ + steps { + mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") + withEnv(["GO_VERSION=1.13.1"]){ + mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") + } + } + } + stage('Functionbeat Mac OS X x-pack'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ON_MACOS != 'false' + } + } + steps { + mageTarget(context: "Functionbeat x-pack Mac OS X", directory: "x-pack/functionbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Functionbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin(context: "Functionbeat Windows Unit test", directory: "x-pack/functionbeat", target: "build unitTest") + } + } + } + } + stage('Journalbeat'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_JOURNALBEAT != "false" + } + } + stages { + stage('Journalbeat oss'){ + steps { + mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build unitTest") + } + } + } + } + stage('Generators'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_GENERATOR != "false" + } + } + stages { + stage('Generators Metricbeat Linux'){ + steps { + makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test") + makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package") + } + } + stage('Generators Beat Linux'){ + steps { + makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test") + makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package") + } + } + stage('Generators Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ON_MACOS != 'false' + } + } + steps { + makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test") + } + post { + always { + delete() + } + } + } + stage('Generators Beat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_ON_MACOS != 'false' + } + } + steps { + makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test") + } + post { + always { + delete() + } + } + } + } + } + stage('Kubernetes'){ + agent { label 'ubuntu-18 && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_KUBERNETES != "false" + } + } + steps { + k8sTest(["v1.18.2","v1.17.2","v1.16.4","v1.15.7","v1.14.10"]) + } } } } } post { - // TODO: Support runbld + always { + runbld() + } cleanup { notifyBuildResult(prComment: true) } } } + +def delete() { + dir("${env.BASE_DIR}") { + fixPermissions("${WORKSPACE}") + } + deleteDir() +} + +def fixPermissions(location) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${location}""", returnStatus: true) +} + +def makeTarget(Map args = [:]) { + def context = args.context + def target = args.target + def directory = args.get('directory', '') + def clean = args.get('clean', true) + def withModule = args.get('withModule', false) + def directoryFlag = directory.trim() ? "-C ${directory}" : '' + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") + whenTrue(clean) { + fixPermissions("${HOME}") + } + } + } +} + +def mageTarget(Map args = [:]) { + def context = args.context + def directory = args.directory + def target = args.target + def withModule = args.get('withModule', false) + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def mageTargetWin(Map args = [:]) { + def context = args.context + def directory = args.directory + def target = args.target + def withModule = args.get('withModule', false) + withGithubNotify(context: "${context}") { + withBeatsEnvWin(withModule: withModule, directory: directory) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMageWin() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def getModulePattern(String toCompare) { + // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') + return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) +} + +def withBeatsEnv(Map args = [:], Closure body) { + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + def modulePattern + if (withModule) { + modulePattern = getModulePattern(directory) + } + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + "MAGEFILE_CACHE=${WORKSPACE}/.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + "PYTHON_ENV=${WORKSPACE}/python-env", + "TEST_TAGS=${env.TEST_TAGS},oracle", + "DOCKER_PULL=0", + "MODULE=${module}" + ]) { + if(isDockerInstalled()){ + dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + } + dir("${env.BASE_DIR}") { + installTools() + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + setGitConfig() + try { + if(!params.dry_run){ + body() + } + } finally { + if (archive) { + archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out') + } + } + } + } +} + +/** + This method archives and report the tests output, for such, it searches in certain folders + to bypass some issues when working with big repositories. +*/ +def archiveTestOutput(Map args = [:]) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + if (isUnix()) { + fixPermissions("${WORKSPACE}") + } + cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') + dir('build') { + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) + } + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") + if (folder.trim()) { + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + tar(file: "${name}.tgz", archive: true, dir: folder) + } + } + } +} + +def withBeatsEnvWin(Map args = [:], Closure body) { + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + def modulePattern + if (withModule) { + modulePattern = getModulePattern(directory) + } + final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", + "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + "MODULE=${module}" + ]){ + dir("${env.BASE_DIR}"){ + installTools() + try { + if(!params.dry_run){ + body() + } + } finally { + archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out") + } + } + } +} + +def installTools() { + def i = 2 // Number of retries + if(isUnix()) { + retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } + retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } + retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } + retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") } + } else { + retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } +} + +def goos(){ + def labels = env.NODE_LABELS + + if (labels.contains('linux')) { + return 'linux' + } else if (labels.contains('windows')) { + return 'windows' + } else if (labels.contains('darwin')) { + return 'darwin' + } + + error("Unhandled OS name in NODE_LABELS: " + labels) +} + +def dumpMage(){ + echo "### MAGE DUMP ###" + sh(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpMageWin(){ + echo "### MAGE DUMP ###" + bat(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpFilteredEnvironment(){ + echo "### ENV DUMP ###" + echo "PATH: ${env.PATH}" + echo "HOME: ${env.HOME}" + echo "USERPROFILE: ${env.USERPROFILE}" + echo "BUILD_DIR: ${env.BUILD_DIR}" + echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" + echo "BEATS: ${env.BEATS}" + echo "PROJECTS: ${env.PROJECTS}" + echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" + echo "PYTHON_ENV: ${env.PYTHON_ENV}" + echo "PYTHON_EXE: ${env.PYTHON_EXE}" + echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" + echo "VENV_PARAMS: ${env.VENV_PARAMS}" + echo "FIND: ${env.FIND}" + echo "GOLINT: ${env.GOLINT}" + echo "GOLINT_REPO: ${env.GOLINT_REPO}" + echo "REVIEWDOG: ${env.REVIEWDOG}" + echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" + echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" + echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" + echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" + echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" + echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" + echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" + echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" + echo "NOW: ${env.NOW}" + echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" + echo "GOIMPORTS: ${env.GOIMPORTS}" + echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" + echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" + echo "PROCESSES: ${env.PROCESSES}" + echo "TIMEOUT: ${env.TIMEOUT}" + echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" + echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" + echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" + echo "STRESS_TESTS: ${env.STRESS_TESTS}" + echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" + echo "TEST_TAGS: ${env.TEST_TAGS}" + echo "GOX_OS: ${env.GOX_OS}" + echo "GOX_OSARCH: ${env.GOX_OSARCH}" + echo "GOX_FLAGS: ${env.GOX_FLAGS}" + echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" + echo "BEAT_VERSION: ${env.BEAT_VERSION}" + echo "COMMIT_ID: ${env.COMMIT_ID}" + echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" + echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" + echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" + echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" + echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" + echo "### END ENV DUMP ###" +} + +def k8sTest(versions){ + versions.each{ v -> + stage("k8s ${v}"){ + withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ + withGithubNotify(context: "K8s ${v}") { + withBeatsEnv(archive: false, withModule: false) { + sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") + sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") + sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") + sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") + sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } + } + } +} + +/** +* isChanged treats the patterns as regular expressions. In order to check if +* any file in a directoy is modified use `^/.*`. +* +* In addition, there are another two alternatives to report that there are +* changes, when `runAllStages` parameter is set to true or when running on a +* branch/tag basis. +*/ +def isChanged(patterns){ + return ( + params.runAllStages // when runAllStages UI parameter is set to true + || !isPR() // when running on a branch/tag + || isGitRegionMatch(patterns: patterns, comparator: 'regexp') + ) +} + +def isChangedOSSCode(patterns) { + def allPatterns = [ + "^Jenkinsfile", + "^go.mod", + "^libbeat/.*", + "^testing/.*", + "^dev-tools/.*", + "^\\.ci/scripts/.*", + ] + allPatterns.addAll(patterns) + return isChanged(allPatterns) +} + +def isChangedXPackCode(patterns) { + def allPatterns = [ + "^Jenkinsfile", + "^go.mod", + "^libbeat/.*", + "^dev-tools/.*", + "^testing/.*", + "^x-pack/libbeat/.*", + "^\\.ci/scripts/.*", + ] + allPatterns.addAll(patterns) + return isChanged(allPatterns) +} + +// withCloudTestEnv executes a closure with credentials for cloud test +// environments. +def withCloudTestEnv(Closure body) { + def maskedVars = [] + def testTags = "${env.TEST_TAGS}" + + // AWS + if (params.allCloudTests || params.awsCloudTests) { + testTags = "${testTags},aws" + def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data + if (!aws.containsKey('access_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") + } + if (!aws.containsKey('secret_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") + } + maskedVars.addAll([ + [var: "AWS_REGION", password: params.awsRegion], + [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], + [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], + ]) + } + + withEnv([ + "TEST_TAGS=${testTags}", + ]) { + withEnvMask(vars: maskedVars) { + body() + } + } +} + +def terraformInit(String directory) { + dir(directory) { + sh(label: "Terraform Init on ${directory}", script: "terraform init") + } +} + +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} + +// Start testing environment on cloud using terraform. Terraform files are +// stashed so they can be used by other stages. They are also archived in +// case manual cleanup is needed. +// +// Example: +// startCloudTestEnv('x-pack-metricbeat', [ +// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], +// ]) +// ... +// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') +def startCloudTestEnv(String name, environments = []) { + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + def runAll = params.runAllCloudTests + try { + for (environment in environments) { + if (environment.cond || runAll) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ + terraformApply(environment.dir) + } + } + } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + } + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') + } + } +} + + +// Looks for all terraform states in directory and runs terraform destroy for them, +// it uses terraform states previously stashed by startCloudTestEnv. +def terraformCleanup(String stashName, String directory) { + stage("Remove cloud scenarios in ${directory}"){ + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + unstash("terraform-${stashName}") + retryWithSleep(retries: 2, seconds: 5, backoff: true) { + sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") + } + } + } + } +} + +def loadConfigEnvVars(){ + def empty = [] + env.GO_VERSION = readFile(".go-version").trim() + + withEnv(["HOME=${env.WORKSPACE}"]) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") } + } + + // Libbeat is the core framework of Beats. It has no additional dependencies + // on other projects in the Beats repository. + env.BUILD_LIBBEAT = isChangedOSSCode(empty) + env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) + + // Auditbeat depends on metricbeat as framework, but does not include any of + // the modules from Metricbeat. + // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. + env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) + env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) + + // Dockerlogbeat is a standalone Beat that only relies on libbeat. + env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) + + // Filebeat depends on libbeat only. + // The Filebeat x-pack build contains all functionality from OSS Filebeat. + env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) + env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) + + // Metricbeat depends on libbeat only. + // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. + env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) + env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) + + // Functionbeat is a standalone beat that depends on libbeat only. + // Functionbeat is available as x-pack build only. + env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) + + // Heartbeat depends on libbeat only. + // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. + env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) + env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) + + // Journalbeat depends on libbeat only. + // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. + env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) + env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) + + // Packetbeat depends on libbeat only. + // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. + env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) + env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) + + // Winlogbeat depends on libbeat only. + // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. + env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) + env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) + + // Elastic-agent is a self-contained product, that depends on libbeat only. + // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. + // The agent is available as x-pack build only. + env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) + + // The Kubernetes test use Filebeat and Metricbeat, but only need to be run + // if the deployment scripts have been updated. No Beats specific testing is + // involved. + env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) + + def generatorPatterns = ['^generator/.*'] + generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) + generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) + env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) + + // Skip all the stages for changes only related to the documentation + env.ONLY_DOCS = isDocChangedOnly() + + // Enable macOS builds when required + env.BUILD_ON_MACOS = (params.macosTest // UI Input parameter is set to true + || !isPR() // For branches and tags + || matchesPrLabel(label: 'macOS') // If `macOS` GH label (Case-Sensitive) + || (env.GITHUB_COMMENT?.toLowerCase()?.contains('/test macos'))) // If `/test macos` in the GH comment (Case-Insensitive) +} + +/** + This method gathers the module name, if required, in order to run the ITs only if + the changeset affects a specific module. + + For such, it's required to look for changes under the module folder and exclude anything else + such as ascidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String pattern, String directory) { + def module = '' + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) + } + return module +} + +/** + This method verifies if the changeset for the current pull request affect only changes related + to documentation, such as asciidoc and png files. +*/ +def isDocChangedOnly(){ + if (params.runAllStages || !env.CHANGE_ID?.trim()) { + log(level: 'INFO', text: 'Speed build for docs only is disabled for branches/tags or when forcing with the runAllStages parameter.') + return 'false' + } else { + log(level: "INFO", text: 'Check if the speed build for docs is enabled.') + return isGitRegionMatch(patterns: ['.*\\.(asciidoc|png)'], shouldMatchAll: true) + } +} + +/** + This method grab the dependencies of a Go module and transform them on regexp +*/ +def getProjectDependencies(beatName){ + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + def output = "" + + withEnv([ + "HOME=${env.WORKSPACE}/${env.BASE_DIR}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + ]) { + output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ + go list -deps ./${beatName} \ + | grep 'elastic/beats' \ + | sed -e "s#github.com/elastic/beats/v7/##g" \ + | awk '{print "^" \$1 "/.*"}' + """) + } + return output?.split('\n').collect{ item -> item as String } +} + +def setGitConfig(){ + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config user.email "beatsmachine@users.noreply.github.com" + git config user.name "beatsmachine" + fi + ''') +} + +def isDockerInstalled(){ + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) +} + +def junitAndStore(Map params = [:]){ + junit(params) + // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds + def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" + stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) + stashedTestReports[stageName] = stageName +} + +def runbld() { + catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { + if (stashedTestReports) { + dir("${env.BASE_DIR}") { + sh(label: 'Prepare workspace context', + script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete') + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) + } + } + sh(label: 'Process JUnit reports with runbld', + script: '''\ + cat >./runbld-script < Date: Tue, 28 Jul 2020 14:32:45 +0100 Subject: [PATCH 008/118] Simplify --- .ci/Jenkinsfile | 57 +++++------------------------------- .ci/jobs/beats-mbp-2.0.yml | 60 -------------------------------------- auditbeat/Jenkinsfile.yml | 25 ++++++++-------- 3 files changed, 20 insertions(+), 122 deletions(-) delete mode 100644 .ci/jobs/beats-mbp-2.0.yml diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 2a6ed9c7c6f0..4b0289189e41 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -80,13 +80,13 @@ pipeline { options { skipDefaultCheckout() } steps { deleteDir() - unstash "source" + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ script { def masterFile = 'Jenkinsfile.yml' def changeset = readYaml(file: masterFile)['changeset'] - readYaml(file: masterFile)['projects'].each { project -> - generateSteps(name: project, changeset: changeset).each { k,v -> + readYaml(file: masterFile)['projects'].each { projectName -> + generateStages(project: projectName, changeset: changeset).each { k,v -> mapParallelTasks["${k}"] = v } } @@ -94,6 +94,11 @@ pipeline { } } } + post { + always { + echo 'TODO: save markdown files for the record' + } + } } } post { @@ -102,50 +107,4 @@ pipeline { notifyBuildResult(prComment: true) } } -} - -def generateSteps(Map args = [:]) { - def name = args.project - def changeset = args.changeset - def mapParallelStages = [:] - def masterFile = "${name}/Jenkinsfile.yml" - def content = readYaml(file: masterFile) - def defaultNode = content['platform'] - - if (beatsWhen(project: name, content: content, changeset: changeset)) { - content['stages'].each { stageName -> - if (stageName['when']) { - if (beatsWhen(project: "stageName", content: stageName['when'], changeset: changeset)) { - generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> - mapParallelStages["${k}"] = v - } - } - } else { - generateStages(defaultNode: defaultNode, content: stageName).each { k,v -> - mapParallelStages["${k}"] = v - } - } - } - return mapParallelStages -} - -def generateStages(Map args = [:]) { - def defaultNode = args.defaultNode - def mapParallelStages = [:] - if (content['platform']) { - content['platform']?.each { - mapParallelStages["${k}-${it}"] = generateStage(platform: it, content: stageName) - } - } else { - mapParallelStages["${k}"] = generateStage(platform: defaultNode, content: stageName) - } - return mapParallelStages -} - -def generateStage(Map args = [:]) { - return { - node(it) { - sh "${content['command']}" - } - } } \ No newline at end of file diff --git a/.ci/jobs/beats-mbp-2.0.yml b/.ci/jobs/beats-mbp-2.0.yml deleted file mode 100644 index d1057dd3047a..000000000000 --- a/.ci/jobs/beats-mbp-2.0.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- job: - name: Beats/beats-mbp-2.0 - display-name: 'Beats (2.0)' - description: 'Beats Main Pipeline 2.0' - view: Beats - concurrent: true - project-type: multibranch - prune-dead-branches: true - days-to-keep: 30 - script-path: '.ci/Jenkinsfile' - triggers: [] - wrappers: [] - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' - discover-tags: true - disable-pr-notifications: true - notification-context: "beats-ci-2.0" - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - skip-initial-build: true - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: 365 - - change-request: - ignore-target-only-changes: true - - named-branches: - - exact-name: - name: 'master' - case-sensitive: true - - regex-name: - regex: '7\.[x789]' - case-sensitive: true - - regex-name: - regex: '8\.\d+' - case-sensitive: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 3 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index f627dfe84761..74454b1b650f 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -1,17 +1,16 @@ when: - branches: true ## for all the branches - tags: true ## for all the tags - changeset: ## when PR contains any of those entries in the changeset + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset - "auditbeat" - - '@oss' ## special token regarding the changeset for the oss - comments: + - '@oss' ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries - "/test auditbeat" - labels: + labels: ## when PR labels matche any of those entries - "auditbeat" - parameters: + parameters: ## when parameter was selected in the UI. - "auditbeat" -platform: ## default platform where the stages will use - - "linux && ubuntu-16" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages stages: build: command: @@ -22,9 +21,9 @@ stages: macos: command: - "cd auditbeat && mage build unitTest" - platform: + platforms: ## override default label in this specific stage. - "macos" - when: ## Aggregate when with the top-level one. + when: ## Aggregate when with the top-level one. comments: - "/test auditbeat for macos" labels: @@ -34,11 +33,11 @@ stages: windows: command: - "cd auditbeat && mage build unitTest" - platform: + platforms: ## override default labels in this specific stage. - "windows-2019" - "windows-2016" when: comments: - "/test auditbeat for windows" parameters: - - "macos" + - "windows" \ No newline at end of file From fcab64b3ba85714424f989880c8b88ae9e7ed89d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 14:50:15 +0100 Subject: [PATCH 009/118] Refactor --- .ci/Jenkinsfile | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4b0289189e41..b014864ae18b 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -83,10 +83,9 @@ pipeline { unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ script { - def masterFile = 'Jenkinsfile.yml' - def changeset = readYaml(file: masterFile)['changeset'] - readYaml(file: masterFile)['projects'].each { projectName -> - generateStages(project: projectName, changeset: changeset).each { k,v -> + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> mapParallelTasks["${k}"] = v } } @@ -107,4 +106,20 @@ pipeline { notifyBuildResult(prComment: true) } } +} + +def generateStages(Map args = [:]) { + def projectName = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def fileName = "${projectName}/Jenkinsfile.yml" + if (fileExists(fileName)) { + def content = readYaml(file: fileName) + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset)) { + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset) + } + } else { + log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") + } + return mapParallelStages } \ No newline at end of file From 6a0f1bd62b3e6448b3fa04042a01e370d324652e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 16:35:25 +0100 Subject: [PATCH 010/118] Add run commands --- .ci/Jenkinsfile | 122 ++++++++++++++++++++++++++++++++++- .ci/scripts/install-tools.sh | 7 ++ Jenkinsfile.yml | 11 +++- auditbeat/Jenkinsfile.yml | 15 +++-- 4 files changed, 144 insertions(+), 11 deletions(-) create mode 100755 .ci/scripts/install-tools.sh diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index b014864ae18b..39f4d0d5cb65 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -83,6 +83,7 @@ pipeline { unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ script { + def mapParallelTasks = [:] def content = readYaml(file: 'Jenkinsfile.yml') content['projects'].each { projectName -> generateStages(project: projectName, changeset: content['changeset']).each { k,v -> @@ -116,10 +117,127 @@ def generateStages(Map args = [:]) { if (fileExists(fileName)) { def content = readYaml(file: fileName) if (beatsWhen(project: projectName, content: content?.when, changeset: changeset)) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset) + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: runCommand()) } } else { log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") } return mapParallelStages -} \ No newline at end of file +} + +def runCommand(Map args = [:]) { + if(args?.containsKey('make')) { + makeTarget(args) + } else { + mageTarget(args) + } +} + +def makeTarget(Map args = [:]) { + def context = args.context + def target = args.target + def directory = args.get('directory', '') + def clean = args.get('clean', true) + def withModule = args.get('withModule', false) + def directoryFlag = directory.trim() ? "-C ${directory}" : '' + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") + } + } +} + +def mageTarget(Map args = [:]) { + def context = args.context + def directory = args.directory + def target = args.target + def withModule = args.get('withModule', false) + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + dir(directory) { + cmd(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def withBeatsEnv(Map args = [:], Closure body) { + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + def modulePattern + if (withModule) { + modulePattern = getModulePattern(directory) + } + def goRoot, path, magefile, pythonEnv, testResults, artifacts + + if(isUnix()) { + def os = goos() + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" + magefile = "${WORKSPACE}/.magefile" + pythonEnv = "${WORKSPACE}/python-env" + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' + } else { + def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}" + magefile = "${env.WORKSPACE}\\.magefile" + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" + } + + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${path}", + "MAGEFILE_CACHE=${magefile}", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + "PYTHON_ENV=${pythonEnv}", + "TEST_TAGS=${env.TEST_TAGS},oracle", + "DOCKER_PULL=0", + "MODULE=${module}" + ]) { + if(isDockerInstalled()){ + dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + } + dir("${env.BASE_DIR}") { + installTools() + + if(isUnix()) { + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config user.email "beatsmachine@users.noreply.github.com" + git config user.name "beatsmachine" + fi''') + } + try { + body() + } finally { + if (archive) { + //archiveTestOutput(testResults: testResults, artifacts: artifacts) + } + } + } + } +} + +def installTools() { + def i = 2 // Number of retries + if(isUnix()) { + retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } else { + retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } +} + diff --git a/.ci/scripts/install-tools.sh b/.ci/scripts/install-tools.sh new file mode 100755 index 000000000000..297a7820cad6 --- /dev/null +++ b/.ci/scripts/install-tools.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -exuo pipefail + +.ci/scripts/install-go.sh +.ci/scripts/install-docker-compose.sh +.ci/scripts/install-terraform.sh +make mage diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index fc602f44219c..c3eed0d1b4d8 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -20,10 +20,17 @@ projects: ## Changeset macros that are defined here and used in each specific 2.0 pipeline. changeset: - oss: + ci: - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: - "^go.mod" - "^libbeat/.*" + - "^dev-tools/.*" - "^testing/.*" + xpack: + - "^go.mod" + - "^libbeat/.*" - "^dev-tools/.*" - - "^\\.ci/scripts/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" \ No newline at end of file diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 74454b1b650f..c9263e8db131 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -2,6 +2,7 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - "auditbeat" + - '@ci' ## special token regarding the changeset for the ci - '@oss' ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test auditbeat" @@ -13,14 +14,14 @@ when: platform: "linux && ubuntu-16" ## default label for all the stages stages: build: - command: - - "cd auditbeat && mage build test" + mage: + - "mage build test" crosscompile: - command: + make: - "make -C auditbeat crosscompile" macos: - command: - - "cd auditbeat && mage build unitTest" + mage: + - "mage build unitTest" platforms: ## override default label in this specific stage. - "macos" when: ## Aggregate when with the top-level one. @@ -31,8 +32,8 @@ stages: parameters: - "macos" windows: - command: - - "cd auditbeat && mage build unitTest" + mage: + - "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - "windows-2016" From a8cc86607fec71705f2988de4ec53552a485b5b5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 16:43:38 +0100 Subject: [PATCH 011/118] use nodeOS step --- .ci/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 39f4d0d5cb65..b5c5eb65b114 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -172,8 +172,7 @@ def withBeatsEnv(Map args = [:], Closure body) { def goRoot, path, magefile, pythonEnv, testResults, artifacts if(isUnix()) { - def os = goos() - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" From 14e039d77d0fcc2c6cdb2af26b8935f1c5c77256 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 17:28:04 +0100 Subject: [PATCH 012/118] Fix GO_VERSION --- .ci/Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index b5c5eb65b114..8580594fc496 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -49,10 +49,12 @@ pipeline { deleteDir() gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - script { - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc and md changes only - env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true) + dir("${BASE_DIR}"){ + // Skip all the stages except docs for PR's with asciidoc and md changes only + setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true)) + setEnvVar('GO_VERSION', readFile(".go-version").trim()) + withEnv(["HOME=${env.WORKSPACE}"]) { + retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } } } } From 7660a6d1a322bba7f473c9dc7c30397a9f82cb41 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 17:49:19 +0100 Subject: [PATCH 013/118] Transform to String --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8580594fc496..1e3106a23425 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ // Skip all the stages except docs for PR's with asciidoc and md changes only - setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true)) + setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) setEnvVar('GO_VERSION', readFile(".go-version").trim()) withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } From 35d30be112bbf3a76bf18f1ea629b3fae847f9ab Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 18:19:06 +0100 Subject: [PATCH 014/118] Use method-pointer-operator --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 1e3106a23425..898e0e80a253 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -119,7 +119,7 @@ def generateStages(Map args = [:]) { if (fileExists(fileName)) { def content = readYaml(file: fileName) if (beatsWhen(project: projectName, content: content?.when, changeset: changeset)) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: runCommand()) + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: this.&runCommand) } } else { log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") From 6be81e2febf50101e16cfff43655c4034df22dbb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 18:27:21 +0100 Subject: [PATCH 015/118] replace with isInstalled --- .ci/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 898e0e80a253..50afdf9be853 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -207,12 +207,11 @@ def withBeatsEnv(Map args = [:], Closure body) { "DOCKER_PULL=0", "MODULE=${module}" ]) { - if(isDockerInstalled()){ + if(isInstalled(tool: 'docker', flag: '--version')) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { installTools() - if(isUnix()) { // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. // See https://github.com/elastic/beats/issues/17787. From e874d5478b92f8bb04d914448ea7b8c6259b24c5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 18:29:51 +0100 Subject: [PATCH 016/118] Fix the YAML data --- auditbeat/Jenkinsfile.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index c9263e8db131..97fbea1b1dee 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -2,8 +2,8 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - "auditbeat" - - '@ci' ## special token regarding the changeset for the ci - - '@oss' ## special token regarding the changeset for the oss + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test auditbeat" labels: ## when PR labels matche any of those entries @@ -23,14 +23,14 @@ stages: mage: - "mage build unitTest" platforms: ## override default label in this specific stage. - - "macos" + - "macosx" when: ## Aggregate when with the top-level one. comments: - "/test auditbeat for macos" labels: - - "macos" + - "macOS" parameters: - - "macos" + - "macosTest" windows: mage: - "mage build unitTest" From 05baf2ebb1cbb4f3ffcf36408679dff29b59f64b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 19:14:41 +0100 Subject: [PATCH 017/118] Use node --- .ci/Jenkinsfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 50afdf9be853..a71d5676111f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -129,35 +129,39 @@ def generateStages(Map args = [:]) { def runCommand(Map args = [:]) { if(args?.containsKey('make')) { - makeTarget(args) + makeTarget(context: args.label, command: args.make, directory: args.project, label: args.label) } else { - mageTarget(args) + mageTarget(context: args.label, command: args.mage, directory: args.project, label: args.label) } } def makeTarget(Map args = [:]) { def context = args.context - def target = args.target + def command = args.command def directory = args.get('directory', '') def clean = args.get('clean', true) def withModule = args.get('withModule', false) def directoryFlag = directory.trim() ? "-C ${directory}" : '' - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") + node(args.label) { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + sh(label: "${command}", script: "${command}") + } } } } def mageTarget(Map args = [:]) { def context = args.context + def command = args.command def directory = args.directory - def target = args.target def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - dir(directory) { - cmd(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + node(args.label) { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + dir(directory) { + cmd(label: "${command}", script: "${command}") + } } } } @@ -233,11 +237,10 @@ def withBeatsEnv(Map args = [:], Closure body) { } def installTools() { - def i = 2 // Number of retries if(isUnix()) { - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } } else { - retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + retryWithSleep(retries: 2, seconds: 5){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } } } From 2a668df2833d565037e227ff8163cb58a17ba7ea Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 28 Jul 2020 19:15:56 +0100 Subject: [PATCH 018/118] sort variables --- .ci/Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a71d5676111f..14c3a654b5f4 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -199,17 +199,17 @@ def withBeatsEnv(Map args = [:], Closure body) { // NOTE: This is required to run after the unstash def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' withEnv([ - "HOME=${env.WORKSPACE}", + "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", "GOROOT=${goRoot}", - "PATH=${path}", + "HOME=${env.WORKSPACE}", "MAGEFILE_CACHE=${magefile}", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", + "MODULE=${module}", + "PATH=${path}", "PYTHON_ENV=${pythonEnv}", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "DOCKER_PULL=0", - "MODULE=${module}" + "RACE_DETECTOR=true", + "TEST_COVERAGE=true", + "TEST_TAGS=${env.TEST_TAGS},oracle" ]) { if(isInstalled(tool: 'docker', flag: '--version')) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") From b22f34adf6878e585b2c30ee3da8f18e2aa8c2d6 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 09:06:02 +0100 Subject: [PATCH 019/118] Use string rather than a list --- auditbeat/Jenkinsfile.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 97fbea1b1dee..68de487f4cff 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -14,14 +14,11 @@ when: platform: "linux && ubuntu-16" ## default label for all the stages stages: build: - mage: - - "mage build test" + mage: "mage build test" crosscompile: - make: - - "make -C auditbeat crosscompile" + make: "make -C auditbeat crosscompile" macos: - mage: - - "mage build unitTest" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "macosx" when: ## Aggregate when with the top-level one. @@ -32,8 +29,7 @@ stages: parameters: - "macosTest" windows: - mage: - - "mage build unitTest" + mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - "windows-2016" From 8126103da25a02925ae48dbc2ef2edc472fe51f3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 09:29:35 +0100 Subject: [PATCH 020/118] [CI] verbose and added some missing functions --- .ci/Jenkinsfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 14c3a654b5f4..6b2aa28271ad 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -128,6 +128,7 @@ def generateStages(Map args = [:]) { } def runCommand(Map args = [:]) { + echo "[INFO] ${args?.toString()}" if(args?.containsKey('make')) { makeTarget(context: args.label, command: args.make, directory: args.project, label: args.label) } else { @@ -244,3 +245,27 @@ def installTools() { } } +def getModulePattern(String toCompare) { + // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') + return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) +} + +/** + This method gathers the module name, if required, in order to run the ITs only if + the changeset affects a specific module. + + For such, it's required to look for changes under the module folder and exclude anything else + such as ascidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String pattern, String directory) { + def module = '' + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) + } + return module +} + From c5a55dbda4e7b5462c5a414233c6c8363d278732 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 10:28:35 +0100 Subject: [PATCH 021/118] Use the right arguments --- .ci/Jenkinsfile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 6b2aa28271ad..a0ea388962a6 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -127,12 +127,22 @@ def generateStages(Map args = [:]) { return mapParallelStages } +/** +* This method is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +* +* What parameters/arguments are supported: +* - label -> the worker labels +* - project -> the name of the project that should match with the folder name. +* - content -> the specific stage data in the /Jenkinsfile.yml +*/ def runCommand(Map args = [:]) { echo "[INFO] ${args?.toString()}" - if(args?.containsKey('make')) { - makeTarget(context: args.label, command: args.make, directory: args.project, label: args.label) - } else { - mageTarget(context: args.label, command: args.mage, directory: args.project, label: args.label) + if(args?.content?.containsKey('make')) { + makeTarget(context: args.label, command: args.content.make, directory: args.project, label: args.label) + } + if(args?.content?.containsKey('mage')) { + mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label) } } From 02e0303864a31be4d5642e0f3bc401e48d94263f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 10:38:15 +0100 Subject: [PATCH 022/118] Use the directory flag in the make --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a0ea388962a6..fd889a5d5e1c 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -156,7 +156,7 @@ def makeTarget(Map args = [:]) { node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - sh(label: "${command}", script: "${command}") + sh(label: "${command}", script: "${command} ${directoryFlag}") } } } From 11653c07d354f0a82975b6b98271c4a07db7476e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 11:31:55 +0100 Subject: [PATCH 023/118] Revert "Use the directory flag in the make" This reverts commit 02e0303864a31be4d5642e0f3bc401e48d94263f. --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index fd889a5d5e1c..a0ea388962a6 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -156,7 +156,7 @@ def makeTarget(Map args = [:]) { node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - sh(label: "${command}", script: "${command} ${directoryFlag}") + sh(label: "${command}", script: "${command}") } } } From 52ec93e315537a8be358c77f33e7bb4952c3a076 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 11:33:32 +0100 Subject: [PATCH 024/118] directoryFlag not required It's already defined in the /Jenkinsfile.yml file --- .ci/Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a0ea388962a6..5011d722ff11 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -137,7 +137,6 @@ def generateStages(Map args = [:]) { * - content -> the specific stage data in the /Jenkinsfile.yml */ def runCommand(Map args = [:]) { - echo "[INFO] ${args?.toString()}" if(args?.content?.containsKey('make')) { makeTarget(context: args.label, command: args.content.make, directory: args.project, label: args.label) } @@ -152,7 +151,6 @@ def makeTarget(Map args = [:]) { def directory = args.get('directory', '') def clean = args.get('clean', true) def withModule = args.get('withModule', false) - def directoryFlag = directory.trim() ? "-C ${directory}" : '' node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { From 14128717b0b0dce55f3163f2b4b617d48bc4f754 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 12:25:18 +0100 Subject: [PATCH 025/118] Archive build reasons --- .ci/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 5011d722ff11..d1569565db11 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -98,7 +98,8 @@ pipeline { } post { always { - echo 'TODO: save markdown files for the record' + // Archive the markdown files that contain the reason for the build to be triggered + archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/*reasons.md") } } } From bd853af1f898f7d632ceac3df494bdcd15ee5084 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 12:25:55 +0100 Subject: [PATCH 026/118] Support windows builds --- auditbeat/Jenkinsfile.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 68de487f4cff..ee52980024aa 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -33,8 +33,3 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2019" - "windows-2016" - when: - comments: - - "/test auditbeat for windows" - parameters: - - "windows" \ No newline at end of file From bc5a6a5cab27ca9c53d9dee1ca0405618c4735ee Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 12:31:32 +0100 Subject: [PATCH 027/118] Tear down environment context in the workers --- .ci/Jenkinsfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index d1569565db11..6533250ff484 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -150,7 +150,6 @@ def makeTarget(Map args = [:]) { def context = args.context def command = args.command def directory = args.get('directory', '') - def clean = args.get('clean', true) def withModule = args.get('withModule', false) node(args.label) { withGithubNotify(context: "${context}") { @@ -241,11 +240,25 @@ def withBeatsEnv(Map args = [:], Closure body) { if (archive) { //archiveTestOutput(testResults: testResults, artifacts: artifacts) } + // Tear down the setup for the permamnent workers. + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + fixPermissions("${WORKSPACE}") + deleteDir() + } } } } } +def fixPermissions(location) { + if(isUnix()) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${location}""", returnStatus: true) + } +} + def installTools() { if(isUnix()) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } From 2df49f855bcfe42ff673a189672905e96a0e1dc0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 12:40:08 +0100 Subject: [PATCH 028/118] [CI] support x-pack/auditbeat and minor changes --- Jenkinsfile.yml | 4 ++-- auditbeat/Jenkinsfile.yml | 2 +- x-pack/auditbeat/Jenkinsfile.yml | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 x-pack/auditbeat/Jenkinsfile.yml diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index c3eed0d1b4d8..2c7d4d00abd2 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -25,12 +25,12 @@ changeset: - "^\\.ci/scripts/.*" oss: - "^go.mod" - - "^libbeat/.*" - "^dev-tools/.*" + - "^libbeat/.*" - "^testing/.*" xpack: - "^go.mod" - - "^libbeat/.*" - "^dev-tools/.*" + - "^libbeat/.*" - "^testing/.*" - "^x-pack/libbeat/.*" \ No newline at end of file diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index ee52980024aa..28f57a165b70 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -1,7 +1,7 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - - "auditbeat" + - "^auditbeat/.*" - "@ci" ## special token regarding the changeset for the ci - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..ad316ada8842 --- /dev/null +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test auditbeat" + labels: ## when PR labels matche any of those entries + - "auditbeat" + parameters: ## when parameter was selected in the UI. + - "auditbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage update build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" + - "windows-2016" From 40723af6bce552be7ffea1cb8b4ea4185502e00f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 13:02:38 +0100 Subject: [PATCH 029/118] Support MODULE env variable --- .ci/Jenkinsfile | 5 +++-- x-pack/auditbeat/Jenkinsfile.yml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 6533250ff484..c6e2a0f5c2f3 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -138,11 +138,12 @@ def generateStages(Map args = [:]) { * - content -> the specific stage data in the /Jenkinsfile.yml */ def runCommand(Map args = [:]) { + def withModule = args.content.get('withModule', false) if(args?.content?.containsKey('make')) { - makeTarget(context: args.label, command: args.content.make, directory: args.project, label: args.label) + makeTarget(context: args.label, command: args.content.make, directory: args.project, label: args.label, withModule: withModule) } if(args?.content?.containsKey('mage')) { - mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label) + mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) } } diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index ad316ada8842..b0fd6ff198fa 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -15,6 +15,7 @@ platform: "linux && ubuntu-18" ## default label for all the stages stages: build: mage: "mage update build test" + withModule: true ## run the ITs only if the changeset affects a specific module. macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From a3d9a5cbf59d373e3ca8ea99f4e6071b0d282d60 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 13:18:44 +0100 Subject: [PATCH 030/118] Build reasons pattern file changed --- .ci/Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index c6e2a0f5c2f3..97d9bf4c97e7 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -98,8 +98,10 @@ pipeline { } post { always { - // Archive the markdown files that contain the reason for the build to be triggered - archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/*reasons.md") + dir("${BASE_DIR}"){ + // Archive the markdown files that contain the build reasons + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.md') + } } } } From 8902166efc6c0d897124bd969e0c1e0627dd9410 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 13:23:45 +0100 Subject: [PATCH 031/118] Support runbld and test reporting --- .ci/Jenkinsfile | 70 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 97d9bf4c97e7..9a2cf2196652 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -2,6 +2,12 @@ @Library('apm@feature/2.0-beats') _ +import groovy.transform.Field +/** + This is required to store the stashed id with the test results to be digested with runbld +*/ +@Field def stashedTestReports = [:] + pipeline { agent { label 'ubuntu-18 && immutable' } environment { @@ -107,7 +113,9 @@ pipeline { } } post { - // TODO: Support runbld + always { + runbld() + } cleanup { notifyBuildResult(prComment: true) } @@ -157,7 +165,7 @@ def makeTarget(Map args = [:]) { node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - sh(label: "${command}", script: "${command}") + cmd(label: "${command}", script: "${command}") } } } @@ -241,7 +249,7 @@ def withBeatsEnv(Map args = [:], Closure body) { body() } finally { if (archive) { - //archiveTestOutput(testResults: testResults, artifacts: artifacts) + archiveTestOutput(testResults: testResults, artifacts: artifacts) } // Tear down the setup for the permamnent workers. catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { @@ -294,3 +302,59 @@ def getCommonModuleInTheChangeSet(String pattern, String directory) { return module } +/** + This method archives and report the tests output, for such, it searches in certain folders + to bypass some issues when working with big repositories. +*/ +def archiveTestOutput(Map args = [:]) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + if (isUnix()) { + fixPermissions("${WORKSPACE}") + } + cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') + dir('build') { + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) + } + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") + if (folder.trim()) { + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + tar(file: "${name}.tgz", archive: true, dir: folder) + } + } + } +} + +def junitAndStore(Map args = [:]){ + junit(args) + // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds + def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" + stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) + stashedTestReports[stageName] = stageName +} + +def runbld() { + catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { + if (stashedTestReports) { + dir("${env.BASE_DIR}") { + sh(label: 'Prepare workspace context', + script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete') + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) + } + } + sh(label: 'Process JUnit reports with runbld', + script: '''\ + cat >./runbld-script < Date: Wed, 29 Jul 2020 14:56:56 +0100 Subject: [PATCH 032/118] [markdown] add PR comments that are supported --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a85c40b31281..65893e69f68e 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ It is possible to trigger some jobs by putting a comment on a GitHub PR. * [beats][] * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build. * `/test macos` will kick off a default build with also the `macos` stages. + * `/test ` will kick off the default build for the given PR in addition to the build itself. + * `/test for macos` will kick off a default build with also the `macos` stage for the . * [apm-beats-update][] * `/run apm-beats-update` From e6acb6f49eb597174a1091d04ab8dd72b47e938d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:05:13 +0100 Subject: [PATCH 033/118] [CI] Update README.md with CI Labels --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65893e69f68e..1a4baa838504 100644 --- a/README.md +++ b/README.md @@ -90,17 +90,25 @@ For testing purposes, we generate snapshot builds that you can find [here](https ## CI +### PR Comments + It is possible to trigger some jobs by putting a comment on a GitHub PR. (This service is only available for users affiliated with Elastic and not for open-source contributors.) * [beats][] * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build. * `/test macos` will kick off a default build with also the `macos` stages. - * `/test ` will kick off the default build for the given PR in addition to the build itself. - * `/test for macos` will kick off a default build with also the `macos` stage for the . + * `/test ` will kick off the default build for the given PR in addition to the `` build itself. + * `/test for macos` will kick off a default build with also the `macos` stage for the ``. * [apm-beats-update][] * `/run apm-beats-update` - [beats]: https://beats-ci.elastic.co/job/Beats/job/beats-beats-mbp/ [apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/ + +### PR Labels + +It's possible to configure the build on a GitHub PR by labelling the PR with the below labels + +* `` to force the following builds to run the stages for the `` +* `macOS` to force the following builds to run the `macos` stages. From 9f2b923cbd8c58a08bb836f1bb1475998772d84a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:18:01 +0100 Subject: [PATCH 034/118] Fix step reference --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9a2cf2196652..6087045ca5ab 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -320,7 +320,7 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() tar(file: "${name}.tgz", archive: true, dir: folder) } } From 09262e5e7ebf1412c72e4a5af5abe946fae66346 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:19:08 +0100 Subject: [PATCH 035/118] Disable tests for the x-pack/auditbeat --- x-pack/auditbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index b0fd6ff198fa..f646fc6217e2 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: build: - mage: "mage update build test" + mage: "mage update build" # TODO: disable test until issues are fixed. withModule: true ## run the ITs only if the changeset affects a specific module. macos: mage: "mage build unitTest" From 6e1e06871e6a1c21b14af85f40de79242c1a62c0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:26:34 +0100 Subject: [PATCH 036/118] Support filebeat --- filebeat/Jenkinsfile.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 filebeat/Jenkinsfile.yml diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml new file mode 100644 index 000000000000..5c86b58f8229 --- /dev/null +++ b/filebeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test filebeat" + labels: ## when PR labels matche any of those entries + - "filebeat" + parameters: ## when parameter was selected in the UI. + - "filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test filebeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" From 08d0d4ae4b2f498a9d8b3a84ac79102a79d285bf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:26:45 +0100 Subject: [PATCH 037/118] Support x-pack/filebeat --- x-pack/filebeat/Jenkinsfile.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 x-pack/filebeat/Jenkinsfile.yml diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml new file mode 100644 index 000000000000..3ccd0c23b757 --- /dev/null +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/filebeat" + labels: ## when PR labels matche any of those entries + - "x-pack-filebeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/filebeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" From 4a6f8d180bc228845a42bfd509a57fc093726453 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:26:56 +0100 Subject: [PATCH 038/118] Support packetbeat --- packetbeat/Jenkinsfile.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 packetbeat/Jenkinsfile.yml diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..8a2e125aaaa5 --- /dev/null +++ b/packetbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^packetbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test packetbeat" + labels: ## when PR labels matche any of those entries + - "packetbeat" + parameters: ## when parameter was selected in the UI. + - "packetbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test packetbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" From 35dfdb4cf0f0efda99d39160033872975a0641d4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 15:36:04 +0100 Subject: [PATCH 039/118] Support metricbeat --- metricbeat/Jenkinsfile.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 metricbeat/Jenkinsfile.yml diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..5cea25e0fed2 --- /dev/null +++ b/metricbeat/Jenkinsfile.yml @@ -0,0 +1,40 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test metricbeat" + labels: ## when PR labels matche any of those entries + - "metricbeat" + parameters: ## when parameter was selected in the UI. + - "metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + unitTest: + mage: "mage build unitTest" + goIntegTest: + mage: "mage goIntegTest" + withModule: true + pythonIntegTest: + mage: "mage pythonIntegTest" + withModule: true + crosscompile: + make: "make -C metricbeat crosscompile" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test metricbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" From f00ca4671252e060c968aa6a689a68d1de998a1b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 29 Jul 2020 17:09:28 +0100 Subject: [PATCH 040/118] [IGNORE] unitTest for the timebeing --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 5cea25e0fed2..e73069b5644d 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: unitTest: - mage: "mage build unitTest" + mage: "mage build" ## Disable unitTest since there are broken tests goIntegTest: mage: "mage goIntegTest" withModule: true From c6c9ea8b81d1e78197c0aa4e0d6a2274a5ec66e6 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 11:07:10 +0100 Subject: [PATCH 041/118] Add configuration for the remaining beats --- Jenkinsfile.yml | 9 ++++-- deploy/kubernetes/Jenkinsfile.yml | 15 ++++++++++ generator/Jenkinsfile.yml | 43 ++++++++++++++++++++++++++++ heartbeat/Jenkinsfile.yml | 32 +++++++++++++++++++++ journalbeat/Jenkinsfile.yml | 17 +++++++++++ libbeat/Jenkinsfile.yml | 20 +++++++++++++ winlogbeat/Jenkinsfile.yml | 21 ++++++++++++++ x-pack/dockerlogbeat/Jenkinsfile.yml | 18 ++++++++++++ x-pack/elastic-agent/Jenkinsfile.yml | 32 +++++++++++++++++++++ x-pack/functionbeat/Jenkinsfile.yml | 32 +++++++++++++++++++++ x-pack/libbeat/Jenkinsfile.yml | 17 +++++++++++ x-pack/metricbeat/Jenkinsfile.yml | 33 +++++++++++++++++++++ x-pack/packetbeat/Jenkinsfile.yml | 20 +++++++++++++ x-pack/winlogbeat/Jenkinsfile.yml | 20 +++++++++++++ 14 files changed, 326 insertions(+), 3 deletions(-) create mode 100644 deploy/kubernetes/Jenkinsfile.yml create mode 100644 generator/Jenkinsfile.yml create mode 100644 heartbeat/Jenkinsfile.yml create mode 100644 journalbeat/Jenkinsfile.yml create mode 100644 libbeat/Jenkinsfile.yml create mode 100644 winlogbeat/Jenkinsfile.yml create mode 100644 x-pack/dockerlogbeat/Jenkinsfile.yml create mode 100644 x-pack/elastic-agent/Jenkinsfile.yml create mode 100644 x-pack/functionbeat/Jenkinsfile.yml create mode 100644 x-pack/libbeat/Jenkinsfile.yml create mode 100644 x-pack/metricbeat/Jenkinsfile.yml create mode 100644 x-pack/packetbeat/Jenkinsfile.yml create mode 100644 x-pack/winlogbeat/Jenkinsfile.yml diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 2c7d4d00abd2..95780826d6b1 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,6 +1,8 @@ projects: - "auditbeat" + - "deploy/kubernetes" - "filebeat" + - "generator" - "heartbeat" - "journalbeat" - "libbeat" @@ -9,14 +11,15 @@ projects: - "winlogbeat" - "x-pack/auditbeat" - "x-pack/dockerlogbeat" + - "x-pack/elastic-agent" - "x-pack/filebeat" - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/journalbeat" - "x-pack/libbeat" - "x-pack/metricbeat" - - "x-pack/packetbeat" - "x-pack/winlogbeat" + ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. ## Changeset macros that are defined here and used in each specific 2.0 pipeline. changeset: diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml new file mode 100644 index 000000000000..71b5bdb1c019 --- /dev/null +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -0,0 +1,15 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^deploy/kubernetes/.*" + comments: ## when PR comment contains any of those entries + - "/test deploy/kubernetes" + labels: ## when PR labels matche any of those entries + - "kubernetes" + parameters: ## when parameter was selected in the UI. + - "kubernetes" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + metricbeat-test: + k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" ## TODO: to be implemented \ No newline at end of file diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml new file mode 100644 index 000000000000..e2ebf371be4a --- /dev/null +++ b/generator/Jenkinsfile.yml @@ -0,0 +1,43 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^generator/.*" + - "generator/common/beatgen" ## TODO getProjectDependencies + - "metricbeat/beater" ## TODO getProjectDependencies + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test generator" + labels: ## when PR labels matche any of those entries + - "generator" + parameters: ## when parameter was selected in the UI. + - "generator" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + metricbeat-test: + make: "make -C generator/_templates/metricbeat test test-package" + beat-test: + make: "make -C generator/_templates/beat test test-package" + macos-metricbeat: + make: "make -C generator/_templates/metricbeat test" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test generator for macos" + labels: + - "macOS" + parameters: + - "macosTest" + macos-beat: + make: "make -C generator/_templates/beat test" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test generator for macos" + labels: + - "macOS" + parameters: + - "macosTest" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..5bf8677635b7 --- /dev/null +++ b/heartbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^heartbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test heartbeat" + labels: ## when PR labels matche any of those entries + - "heartbeat" + parameters: ## when parameter was selected in the UI. + - "heartbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test heartbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..8f4877576f10 --- /dev/null +++ b/journalbeat/Jenkinsfile.yml @@ -0,0 +1,17 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^journalbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test journalbeat" + labels: ## when PR labels matche any of those entries + - "journalbeat" + parameters: ## when parameter was selected in the UI. + - "journalbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + unitTest: + mage: "mage build unitTest" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..13ba2015fc37 --- /dev/null +++ b/libbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test libbeat" + labels: ## when PR labels matche any of those entries + - "libbeat" + parameters: ## when parameter was selected in the UI. + - "libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + crosscompile: + make: "make -C libbeat crosscompile" + stress-tests: + make: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..e2a185ffffa0 --- /dev/null +++ b/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test winlogbeat" + labels: ## when PR labels matche any of those entries + - "winlogbeat" + parameters: ## when parameter was selected in the UI. + - "winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + crosscompile: + make: "make -C winlogbeat crosscompile" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..5a4dfe68908f --- /dev/null +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -0,0 +1,18 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/dockerlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/dockerlogbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-dockerlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-dockerlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml new file mode 100644 index 000000000000..91fc0e847090 --- /dev/null +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/elastic-agent/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/elastic-agent" + labels: ## when PR labels matche any of those entries + - "x-pack-elastic-agent" + parameters: ## when parameter was selected in the UI. + - "x-pack-elastic-agent" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/elastic-agent for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..ec62a2bb417e --- /dev/null +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/functionbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/functionbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-functionbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-functionbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: ## TODO: missing testGCPFunctions + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/functionbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..a7c217d4bc6d --- /dev/null +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -0,0 +1,17 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/libbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/libbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-libbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..2a4a5f3fc259 --- /dev/null +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/metricbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-metricbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/metricbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..01dfe0e11c60 --- /dev/null +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + windows: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..01dfe0e11c60 --- /dev/null +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + windows: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019" From 3ebe3e08e45a1fe5ca31f30624a04762d08c78f7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 11:17:41 +0100 Subject: [PATCH 042/118] Refactor getCommonModuleInTheChangeSet --- .ci/Jenkinsfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 6087045ca5ab..711ca54d6edc 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -191,10 +191,7 @@ def withBeatsEnv(Map args = [:], Closure body) { def archive = args.get('archive', true) def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) - } + def goRoot, path, magefile, pythonEnv, testResults, artifacts if(isUnix()) { @@ -217,7 +214,7 @@ def withBeatsEnv(Map args = [:], Closure body) { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' withEnv([ "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", @@ -278,11 +275,6 @@ def installTools() { } } -def getModulePattern(String toCompare) { - // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') - return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) -} - /** This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. @@ -290,8 +282,11 @@ def getModulePattern(String toCompare) { For such, it's required to look for changes under the module folder and exclude anything else such as ascidoc and png files. */ -def getCommonModuleInTheChangeSet(String pattern, String directory) { +def getCommonModuleInTheChangeSet(String directory) { + // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') + def pattern = (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) def module = '' + // Transform folder structure in regex format since path separator is required to be escaped def transformedDirectory = directory.replaceAll('/', '\\/') def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" From aa672c3732e3dcf6b6fcfecd5656a84ecdfca63d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 11:24:43 +0100 Subject: [PATCH 043/118] [CI] Cosmetic change --- .ci/Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 711ca54d6edc..03b122acf45a 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -106,7 +106,7 @@ pipeline { always { dir("${BASE_DIR}"){ // Archive the markdown files that contain the build reasons - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.md') + archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md') } } } @@ -155,6 +155,9 @@ def runCommand(Map args = [:]) { if(args?.content?.containsKey('mage')) { mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) } + if(args?.content?.containsKey('k8sTest')) { + log(level: 'WARN', text: "TBD: k8sTest") + } } def makeTarget(Map args = [:]) { From abd658930815184b73b47037844fe97e369bff8a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 12:03:59 +0100 Subject: [PATCH 044/118] Fix leftover and add comments --- .ci/Jenkinsfile | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 03b122acf45a..5c9afdf3a70e 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -122,6 +122,10 @@ pipeline { } } +/** +* This method is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +*/ def generateStages(Map args = [:]) { def projectName = args.project def changeset = args.changeset @@ -190,6 +194,9 @@ def mageTarget(Map args = [:]) { } } +/** +* This method wraps all the environment setup and pre-requirements to run any commands. +*/ def withBeatsEnv(Map args = [:], Closure body) { def archive = args.get('archive', true) def withModule = args.get('withModule', false) @@ -261,15 +268,25 @@ def withBeatsEnv(Map args = [:], Closure body) { } } +/** +* This method fixes the filesystem permissions after the build has happenend. The reason is to +* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental +* issues. +*/ def fixPermissions(location) { if(isUnix()) { sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + set +x source ./dev-tools/common.bash docker_setup script/fix_permissions.sh ${location}""", returnStatus: true) } } +/** +* This method installs the required dependencies that are for some reason not available in the +* CI Workers. +*/ def installTools() { if(isUnix()) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } @@ -279,15 +296,15 @@ def installTools() { } /** - This method gathers the module name, if required, in order to run the ITs only if - the changeset affects a specific module. - - For such, it's required to look for changes under the module folder and exclude anything else - such as ascidoc and png files. +* This method gathers the module name, if required, in order to run the ITs only if +* the changeset affects a specific module. +* +* For such, it's required to look for changes under the module folder and exclude anything else +* such as ascidoc and png files. */ def getCommonModuleInTheChangeSet(String directory) { // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') - def pattern = (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) + def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) def module = '' // Transform folder structure in regex format since path separator is required to be escaped @@ -301,8 +318,8 @@ def getCommonModuleInTheChangeSet(String directory) { } /** - This method archives and report the tests output, for such, it searches in certain folders - to bypass some issues when working with big repositories. +* This method archives and report the tests output, for such, it searches in certain folders +* to bypass some issues when working with big repositories. */ def archiveTestOutput(Map args = [:]) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { @@ -325,6 +342,10 @@ def archiveTestOutput(Map args = [:]) { } } +/** +* This method wraps the junit built-in step to archive the test reports that gonna be populated later on +* with the runbld post build step. +*/ def junitAndStore(Map args = [:]){ junit(args) // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds @@ -333,6 +354,10 @@ def junitAndStore(Map args = [:]){ stashedTestReports[stageName] = stageName } +/** +* This method populates the test output using the runbld approach. For such it requires the +* global variable stashedTestReports. +*/ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { From 1d4ea16e23d82a8de204a0193f0115624ca7d24a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 13:50:26 +0100 Subject: [PATCH 045/118] Fix leftovers --- libbeat/Jenkinsfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 13ba2015fc37..700cb6ad2785 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -13,8 +13,8 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: build: - mage: "mage build test" + mage: "mage build" ## TODO test goal has been disabled since there are broken tests. crosscompile: make: "make -C libbeat crosscompile" stress-tests: - make: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" + make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" From 1e3d9b290a76f385c4b912c4307006dfbc5dab4c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 14:22:03 +0100 Subject: [PATCH 046/118] Disable linting for the time being with some TODO --- .ci/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 5c9afdf3a70e..05b0214de285 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -81,7 +81,9 @@ pipeline { } } steps { - echo 'linting' + whenFalse(true) { // TODO: disable for the time being. + makeTarget(context: "Lint", target: "check") + } } } stage('Build&Test') { From 4e1d3a156bc90898a48b15d08e9f27c036deb2d8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 14:22:19 +0100 Subject: [PATCH 047/118] [CI] Script to generate the build-stage table --- .ci/scripts/generate_build_table.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 .ci/scripts/generate_build_table.py diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py new file mode 100755 index 000000000000..a2c7fe28a8d2 --- /dev/null +++ b/.ci/scripts/generate_build_table.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import os +import yaml + +if __name__ == "__main__": + + print("| Beat | Stage | Command | MODULE | Platforms |") + print("|-------|--------|----------|---------|------------|") + for root, dirs, files in os.walk("."): + dirs.sort() + for file in files: + if file.endswith("Jenkinsfile.yml") and root != ".": + with open(os.path.join(root, file), 'r') as f: + doc = yaml.load(f, Loader=yaml.FullLoader) + module = root.replace(".{}".format(os.sep), '') + platforms = [ doc["platform"] ] + for stage in doc["stages"]: + withModule = False + if "make" in doc["stages"][stage]: + command = doc["stages"][stage]["make"] + if "mage" in doc["stages"][stage]: + command = doc["stages"][stage]["mage"] + if "platforms" in doc["stages"][stage]: + platforms = doc["stages"][stage]["platforms"] + if "withModule" in doc["stages"][stage]: + withModule = doc["stages"][stage]["withModule"] + print("| {} | {} | `{}` | {} | `{}` |".format(module, stage, command, withModule, platforms)) From 4b8466bd63678a8e0eacacbe9a02e6301f965415 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 14:50:11 +0100 Subject: [PATCH 048/118] [docs] Add when in the table markdown --- .ci/scripts/generate_build_table.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py index a2c7fe28a8d2..d19d82888ab6 100755 --- a/.ci/scripts/generate_build_table.py +++ b/.ci/scripts/generate_build_table.py @@ -5,8 +5,8 @@ if __name__ == "__main__": - print("| Beat | Stage | Command | MODULE | Platforms |") - print("|-------|--------|----------|---------|------------|") + print("| Beat | Stage | Command | MODULE | Platforms | When |") + print("|-------|--------|----------|---------|------------|------|") for root, dirs, files in os.walk("."): dirs.sort() for file in files: @@ -17,6 +17,7 @@ platforms = [ doc["platform"] ] for stage in doc["stages"]: withModule = False + when = "Always" if "make" in doc["stages"][stage]: command = doc["stages"][stage]["make"] if "mage" in doc["stages"][stage]: @@ -25,4 +26,6 @@ platforms = doc["stages"][stage]["platforms"] if "withModule" in doc["stages"][stage]: withModule = doc["stages"][stage]["withModule"] - print("| {} | {} | `{}` | {} | `{}` |".format(module, stage, command, withModule, platforms)) + if "when" in doc["stages"][stage]: + when = "some cases" + print("| {} | {} | `{}` | {} | `{}` | `{}` |".format(module, stage, command, withModule, platforms, when)) From a361e8de6fc3ba544fd886577c4ebc6abd26f899 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 15:03:27 +0100 Subject: [PATCH 049/118] [CI] support deploy/kubernetes --- .ci/Jenkinsfile | 31 ++++++++++++++++++++++++++++--- deploy/kubernetes/Jenkinsfile.yml | 4 ++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 05b0214de285..32c9d95ab284 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -162,7 +162,32 @@ def runCommand(Map args = [:]) { mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) } if(args?.content?.containsKey('k8sTest')) { - log(level: 'WARN', text: "TBD: k8sTest") + k8sTest(versions: args.content.k8sTest.split('-'), label: args.label) + } +} + +def k8sTest(map args = [:]) { + def versions = args.versions + node(args.label) { + versions.each{ v -> + stage("k8s ${v}"){ + withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ + withGithubNotify(context: "K8s ${v}") { + withBeatsEnv(archive: false, withModule: false) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } + try { + sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") + sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") + sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") + } finally { + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } + } + } + } } } @@ -291,9 +316,9 @@ def fixPermissions(location) { */ def installTools() { if(isUnix()) { - retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } } else { - retryWithSleep(retries: 2, seconds: 5){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } } } diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 71b5bdb1c019..532523cc90f9 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -11,5 +11,5 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - metricbeat-test: - k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" ## TODO: to be implemented \ No newline at end of file + k8sTest: + k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" \ No newline at end of file From ce7e02b123c0d6dd6712cbe7a24fd797bbf0c075 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 15:33:31 +0100 Subject: [PATCH 050/118] [CI] support cloud aws for x-pack/metricbeats --- .ci/Jenkinsfile | 96 +++++++++++++++++++++++++++++++ x-pack/metricbeat/Jenkinsfile.yml | 4 ++ 2 files changed, 100 insertions(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 32c9d95ab284..36a89a988bc3 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -164,6 +164,22 @@ def runCommand(Map args = [:]) { if(args?.content?.containsKey('k8sTest')) { k8sTest(versions: args.content.k8sTest.split('-'), label: args.label) } + if(args?.content?.containsKey('cloud')) { + cloud(context: args.label, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule) + } +} + +def cloud(map args = [:]) { + node(args.label) { + stage("${args.contex}-prepare-cloud-env"){ + startCloudTestEnv(args.context, dirs: ['x-pack/metricbeat/module/aws']) + } + stage("${args.contex}-prepare-cloud-env"){ + withCloudTestEnv() { + mageTarget(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule) + } + } + } } def k8sTest(map args = [:]) { @@ -408,3 +424,83 @@ def runbld() { } } } + +/** +* This method executes a closure with credentials for cloud test +* environments. +*/ +def withCloudTestEnv(Closure body) { + def maskedVars = [] + def testTags = "${env.TEST_TAGS}" + + // AWS + if (params.allCloudTests || params.awsCloudTests) { + testTags = "${testTags},aws" + def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data + if (!aws.containsKey('access_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") + } + if (!aws.containsKey('secret_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") + } + maskedVars.addAll([ + [var: "AWS_REGION", password: params.awsRegion], + [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], + [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], + ]) + } + + withEnv([ + "TEST_TAGS=${testTags}", + ]) { + withEnvMask(vars: maskedVars) { + body() + } + } +} + +/** +* Start testing environment on cloud using terraform. Terraform files are +* stashed so they can be used by other stages. They are also archived in +* case manual cleanup is needed. +* +* Example: +* startCloudTestEnv('x-pack-metricbeat', dir: ['x-pack/metricbeat/module/aws']) +* ... +* terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') +*/ +def startCloudTestEnv(String name, dirs = []) { + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + try { + for (folder in dirs) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ + terraformApply(folder) + } + } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + } + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') + } + } +} + +/** +* Tear down the terraform environments, by looking for all terraform states in directory +* then it runs terraform destroy for each one. +* it uses terraform states previously stashed by startCloudTestEnv. +*/ +def terraformCleanup(String stashName, String directory) { + stage("Remove cloud scenarios in ${directory}"){ + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + unstash("terraform-${stashName}") + retryWithSleep(retries: 2, seconds: 5, backoff: true) { + sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 2a4a5f3fc259..0bfc80317d31 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -16,6 +16,10 @@ stages: build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + when: ## Aggregate when with the top-level one. + parameters: + - "awsCloudTests" + - "runAllCloudTests" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From f2cdc17e181464cf9cc34f2e781ce40d58765bcc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 15:34:22 +0100 Subject: [PATCH 051/118] Fix typo --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 36a89a988bc3..7b4adf37032d 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -169,7 +169,7 @@ def runCommand(Map args = [:]) { } } -def cloud(map args = [:]) { +def cloud(Map args = [:]) { node(args.label) { stage("${args.contex}-prepare-cloud-env"){ startCloudTestEnv(args.context, dirs: ['x-pack/metricbeat/module/aws']) @@ -182,7 +182,7 @@ def cloud(map args = [:]) { } } -def k8sTest(map args = [:]) { +def k8sTest(Map args = [:]) { def versions = args.versions node(args.label) { versions.each{ v -> From 38ea5dd9bb4341f1816906669a19289ca86c4cc1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 15:38:44 +0100 Subject: [PATCH 052/118] [CI] It uses another node --- .ci/Jenkinsfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 7b4adf37032d..463ed5623803 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -174,11 +174,9 @@ def cloud(Map args = [:]) { stage("${args.contex}-prepare-cloud-env"){ startCloudTestEnv(args.context, dirs: ['x-pack/metricbeat/module/aws']) } - stage("${args.contex}-prepare-cloud-env"){ - withCloudTestEnv() { - mageTarget(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule) - } - } + } + withCloudTestEnv() { + mageTarget(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule) } } From f3beedc2c3359d2b9a8a7523c77cce73ae8b853d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 15:43:39 +0100 Subject: [PATCH 053/118] Use context that's populated from the beatsStages step --- .ci/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 463ed5623803..5947452ab302 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -156,16 +156,16 @@ def generateStages(Map args = [:]) { def runCommand(Map args = [:]) { def withModule = args.content.get('withModule', false) if(args?.content?.containsKey('make')) { - makeTarget(context: args.label, command: args.content.make, directory: args.project, label: args.label, withModule: withModule) + makeTarget(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule) } if(args?.content?.containsKey('mage')) { - mageTarget(context: args.label, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) + mageTarget(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) } if(args?.content?.containsKey('k8sTest')) { - k8sTest(versions: args.content.k8sTest.split('-'), label: args.label) + k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label) } if(args?.content?.containsKey('cloud')) { - cloud(context: args.label, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule) + cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule) } } @@ -184,9 +184,9 @@ def k8sTest(Map args = [:]) { def versions = args.versions node(args.label) { versions.each{ v -> - stage("k8s ${v}"){ + stage("${args.context} ${v}"){ withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "K8s ${v}") { + withGithubNotify(context: "${args.context} ${v}") { withBeatsEnv(archive: false, withModule: false) { retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } From f99006ceccc7c5dac4e641167ae198effdf9dddb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:05:59 +0100 Subject: [PATCH 054/118] [DOCS] Add when column with some legend --- .ci/scripts/generate_build_table.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py index d19d82888ab6..b702019757c4 100755 --- a/.ci/scripts/generate_build_table.py +++ b/.ci/scripts/generate_build_table.py @@ -15,9 +15,21 @@ doc = yaml.load(f, Loader=yaml.FullLoader) module = root.replace(".{}".format(os.sep), '') platforms = [ doc["platform"] ] + when = "" + if "branches" in doc["when"]: + when = f"{when}/:palm_tree:" + if "changeset" in doc["when"]: + when = f"{when}/:file_folder:" + if "comments" in doc["when"]: + when = f"{when}/:speech_balloon:" + if "labels" in doc["when"]: + when = f"{when}/:label:" + if "parameters" in doc["when"]: + when = f"{when}/:smiley:" + if "tags" in doc["when"]: + when = f"{when}/:taco:" for stage in doc["stages"]: withModule = False - when = "Always" if "make" in doc["stages"][stage]: command = doc["stages"][stage]["make"] if "mage" in doc["stages"][stage]: @@ -27,5 +39,13 @@ if "withModule" in doc["stages"][stage]: withModule = doc["stages"][stage]["withModule"] if "when" in doc["stages"][stage]: - when = "some cases" - print("| {} | {} | `{}` | {} | `{}` | `{}` |".format(module, stage, command, withModule, platforms, when)) + when = f"{when}/:star:" + print("| {} | {} | `{}` | {} | `{}` | {} |".format(module, stage, command, withModule, platforms, when)) + +print("> :palm_tree: -> Git Branch based") +print("> :label: -> GitHub Pull Request Label based") +print("> :file_folder: -> Changeset based") +print("> :speech_balloon: -> GitHub Pull Request comment based") +print("> :taco: -> Git tag based") +print("> :smiley: -> Manual UI interaction based") +print("> :star: -> More specific cases based") From eff00d7a27aa630463618eb584d8260d90be8b66 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:09:40 +0100 Subject: [PATCH 055/118] No yet supported --- x-pack/winlogbeat/Jenkinsfile.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 x-pack/winlogbeat/Jenkinsfile.yml diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml deleted file mode 100644 index 01dfe0e11c60..000000000000 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ /dev/null @@ -1,20 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/winlogbeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@xpack" ## special token regarding the changeset for the xpack - comments: ## when PR comment contains any of those entries - - "/test x-pack/winlogbeat" - labels: ## when PR labels matche any of those entries - - "x-pack-winlogbeat" - parameters: ## when parameter was selected in the UI. - - "x-pack-winlogbeat" - tags: true ## for all the tags -platform: "linux && ubuntu-18" ## default label for all the stages -stages: - windows: - mage: "mage build unitTest" - withModule: true - platforms: ## override default labels in this specific stage. - - "windows-2019" From d0d8d73a29df6a48e1a5502425863ec5149e334a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:11:27 +0100 Subject: [PATCH 056/118] Enable AWS testing for the time being --- .ci/Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 5947452ab302..62d3dc382b84 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -40,12 +40,10 @@ pipeline { } parameters { booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') } stages { stage('Checkout') { @@ -82,7 +80,7 @@ pipeline { } steps { whenFalse(true) { // TODO: disable for the time being. - makeTarget(context: "Lint", target: "check") + makeTarget(context: "Lint", target: 'make check') } } } From 6e381fa4ed3aa3e6edb459209b399f479e67cb63 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:24:06 +0100 Subject: [PATCH 057/118] For debugging purposes --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 62d3dc382b84..368fcad14ede 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - PIPELINE_LOG_LEVEL = 'INFO' + PIPELINE_LOG_LEVEL = 'DEBUG' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' RUNBLD_DISABLE_NOTIFICATIONS = 'true' TERRAFORM_VERSION = "0.12.24" From d80a49073bef3331c4d3e0511a9418ee1e643bd2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:44:35 +0100 Subject: [PATCH 058/118] Dump and support x-pack/winlogbeat --- .ci/Jenkinsfile | 66 ++++++++++++++++++++++++++++++- x-pack/winlogbeat/Jenkinsfile.yml | 20 ++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 x-pack/winlogbeat/Jenkinsfile.yml diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 368fcad14ede..56c12c7089e7 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -211,6 +211,7 @@ def makeTarget(Map args = [:]) { node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + dumpVariables() cmd(label: "${command}", script: "${command}") } } @@ -225,6 +226,7 @@ def mageTarget(Map args = [:]) { node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + dumpVariables() dir(directory) { cmd(label: "${command}", script: "${command}") } @@ -499,4 +501,66 @@ def terraformCleanup(String stashName, String directory) { } } } -} \ No newline at end of file +} + +/** +* For debugging purposes. +*/ +def dumpVariables(){ + echo "### MAGE DUMP ###" + cmd(label: 'Dump mage variables', script: 'mage dumpVariables') + echo "### END MAGE DUMP ###" + echo """ + ### ENV DUMP ### + BEAT_VERSION: ${env.BEAT_VERSION} + BEATS: ${env.BEATS} + BUILD_DIR: ${env.BUILD_DIR} + COMMIT_ID: ${env.COMMIT_ID} + COVERAGE_DIR: ${env.COVERAGE_DIR} + COVERAGE_TOOL: ${env.COVERAGE_TOOL} + COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO} + DOCKER_CACHE: ${env.DOCKER_CACHE} + DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME} + DOCKER_COMPOSE: ${env.DOCKER_COMPOSE} + FIND: ${env.FIND} + GOBUILD_FLAGS: ${env.GOBUILD_FLAGS} + GOIMPORTS: ${env.GOIMPORTS} + GOIMPORTS_REPO: ${env.GOIMPORTS_REPO} + GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX} + GOLINT: ${env.GOLINT} + GOLINT_REPO: ${env.GOLINT_REPO} + GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP} + GOX_FLAGS: ${env.GOX_FLAGS} + GOX_OS: ${env.GOX_OS} + GOX_OSARCH: ${env.GOX_OSARCH} + HOME: ${env.HOME} + NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS} + NOW: ${env.NOW} + PATH: ${env.PATH} + PKG_BUILD_DIR: ${env.PKG_BUILD_DIR} + PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR} + PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS} + PROJECTS: ${env.PROJECTS} + PROJECTS_ENV: ${env.PROJECTS_ENV} + PYTHON_ENV: ${env.PYTHON_ENV} + PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE} + PYTHON_EXE: ${env.PYTHON_EXE} + PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES} + PROCESSES: ${env.PROCESSES} + REVIEWDOG: ${env.REVIEWDOG} + REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS} + REVIEWDOG_REPO: ${env.REVIEWDOG_REPO} + STRESS_TESTS: ${env.STRESS_TESTS} + STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS} + SYSTEM_TESTS: ${env.SYSTEM_TESTS} + TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO} + TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT} + TEST_TAGS: ${env.TEST_TAGS} + TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT} + TIMEOUT: ${env.TIMEOUT} + USERPROFILE: ${env.USERPROFILE} + VENV_PARAMS: ${env.VENV_PARAMS} + XPACK_SUFFIX: ${env.XPACK_SUFFIX} + ### END ENV DUMP ### + """ +} diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..d6dc55c04fb7 --- /dev/null +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matche any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "windows-2019" ## default label for all the stages +stages: + build: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019" From f9bd2270fb6a2996578ed2d3c179417d2205e350 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 16:51:10 +0100 Subject: [PATCH 059/118] Refactor makeTarget and mageTarget with target --- .ci/Jenkinsfile | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 56c12c7089e7..5fcf82bb4785 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -80,7 +80,7 @@ pipeline { } steps { whenFalse(true) { // TODO: disable for the time being. - makeTarget(context: "Lint", target: 'make check') + target(context: "Lint", target: 'make check') } } } @@ -154,10 +154,10 @@ def generateStages(Map args = [:]) { def runCommand(Map args = [:]) { def withModule = args.content.get('withModule', false) if(args?.content?.containsKey('make')) { - makeTarget(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule) + target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false) } if(args?.content?.containsKey('mage')) { - mageTarget(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule) + target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true) } if(args?.content?.containsKey('k8sTest')) { k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label) @@ -174,7 +174,7 @@ def cloud(Map args = [:]) { } } withCloudTestEnv() { - mageTarget(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule) + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true) } } @@ -203,31 +203,24 @@ def k8sTest(Map args = [:]) { } } -def makeTarget(Map args = [:]) { +/** +* This method runs the given command supporting two kind of scenarios: +* - make -C then the dir(location) is not required, aka by disaling isMage: false +* - mage then the dir(location) is required, aka by enabling isMage: true. +*/ +def target(Map args = [:]) { def context = args.context def command = args.command def directory = args.get('directory', '') def withModule = args.get('withModule', false) + def isMage = args.get('isMage', false) node(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory) { dumpVariables() - cmd(label: "${command}", script: "${command}") - } - } - } -} - -def mageTarget(Map args = [:]) { - def context = args.context - def command = args.command - def directory = args.directory - def withModule = args.get('withModule', false) - node(args.label) { - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - dumpVariables() - dir(directory) { + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { cmd(label: "${command}", script: "${command}") } } @@ -344,7 +337,7 @@ def installTools() { * such as ascidoc and png files. */ def getCommonModuleInTheChangeSet(String directory) { - // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') + // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) def module = '' From 39364301bdbc235fc59eede76d445730a758e6ed Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 17:08:25 +0100 Subject: [PATCH 060/118] Fix typo in the variable name --- .ci/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 5fcf82bb4785..a5479ddc3cb9 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -150,6 +150,7 @@ def generateStages(Map args = [:]) { * - label -> the worker labels * - project -> the name of the project that should match with the folder name. * - content -> the specific stage data in the /Jenkinsfile.yml +* - context -> the name of the stage, normally -(-)? */ def runCommand(Map args = [:]) { def withModule = args.content.get('withModule', false) @@ -169,7 +170,7 @@ def runCommand(Map args = [:]) { def cloud(Map args = [:]) { node(args.label) { - stage("${args.contex}-prepare-cloud-env"){ + stage("${args.context}-prepare-cloud-env"){ startCloudTestEnv(args.context, dirs: ['x-pack/metricbeat/module/aws']) } } From ae322e3ec768cbb74acf07a96489874b215d52aa Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 17:31:52 +0100 Subject: [PATCH 061/118] Support getProjectDependencies in the beatsWhen --- .ci/Jenkinsfile | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a5479ddc3cb9..6ba8c845277f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -133,7 +133,8 @@ def generateStages(Map args = [:]) { def fileName = "${projectName}/Jenkinsfile.yml" if (fileExists(fileName)) { def content = readYaml(file: fileName) - if (beatsWhen(project: projectName, content: content?.when, changeset: changeset)) { + // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: this.&getProjectDependencies)) { mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: this.&runCommand) } } else { @@ -497,6 +498,31 @@ def terraformCleanup(String stashName, String directory) { } } +/** +* This method fetchs the dependencies of a Go module for such it transforms them in a +* regex pattern. +*/ +def getProjectDependencies(Map args = [:]) { + def beatName = args.project + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + def output = "" + + dir("${env.BASE_DIR}") { + withEnv([ + "HOME=${env.WORKSPACE}/${env.BASE_DIR}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + ]) { + output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ + go list -deps ./${beatName} \ + | grep 'elastic/beats' \ + | sed -e "s#github.com/elastic/beats/v7/##g" \ + | awk '{print "^" \$1 "/.*"}' + """) + } + } + return output?.split('\n').collect{ item -> item as String } +} + /** * For debugging purposes. */ From 7b8822c98fcd1fe32952fea7a058e0ad3b23d664 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 17:33:30 +0100 Subject: [PATCH 062/118] no debug in the CI for now --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 6ba8c845277f..41c9efae97f5 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - PIPELINE_LOG_LEVEL = 'DEBUG' + PIPELINE_LOG_LEVEL = 'INFO' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' RUNBLD_DISABLE_NOTIFICATIONS = 'true' TERRAFORM_VERSION = "0.12.24" From 83efbc5b5f5a0b08af9b9c0c65481d189cb4ffc9 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Jul 2020 17:44:21 +0100 Subject: [PATCH 063/118] Terraform leftovers in the CI --- .ci/Jenkinsfile | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 41c9efae97f5..3d59867836da 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -172,11 +172,15 @@ def runCommand(Map args = [:]) { def cloud(Map args = [:]) { node(args.label) { stage("${args.context}-prepare-cloud-env"){ - startCloudTestEnv(args.context, dirs: ['x-pack/metricbeat/module/aws']) + startCloudTestEnv(name: args.directory, dirs: ['x-pack/metricbeat/module/aws']) } } withCloudTestEnv() { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true) + try { + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true) + } finally { + terraformCleanup(name: args.directory, dir: args.directory) + } } } @@ -458,11 +462,13 @@ def withCloudTestEnv(Closure body) { * case manual cleanup is needed. * * Example: -* startCloudTestEnv('x-pack-metricbeat', dir: ['x-pack/metricbeat/module/aws']) +* startCloudTestEnv(name: 'x-pack-metricbeat', dirs: ['x-pack/metricbeat/module/aws']) * ... -* terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') +* terraformCleanup(name: 'x-pack-metricbeat', dir: 'x-pack/metricbeat') */ -def startCloudTestEnv(String name, dirs = []) { +def startCloudTestEnv(Map args = [:]) { + String name = normalise(args.name) + def dirs = args.get('dirs',[]) withCloudTestEnv() { withBeatsEnv(archive: false, withModule: false) { try { @@ -485,11 +491,13 @@ def startCloudTestEnv(String name, dirs = []) { * then it runs terraform destroy for each one. * it uses terraform states previously stashed by startCloudTestEnv. */ -def terraformCleanup(String stashName, String directory) { +def terraformCleanup(Map args = [:]) { + String name = normalise(args.name) + String directory = args.dir stage("Remove cloud scenarios in ${directory}"){ withCloudTestEnv() { withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${stashName}") + unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") } @@ -498,6 +506,32 @@ def terraformCleanup(String stashName, String directory) { } } +/** +* Prepare the terraform context in the given directory +*/ +def terraformInit(String directory) { + dir(directory) { + sh(label: "Terraform Init on ${directory}", script: "terraform init") + } +} + +/** +* Run terraform in the given directory +*/ +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} + +/** +* Replace the slashes in the directory in case there are nested folders. +*/ +def normalise(String directory) { + return directory.replaceAll("[\\W]|_",'-') +} + /** * This method fetchs the dependencies of a Go module for such it transforms them in a * regex pattern. From 76a90ae631ab2643ff51cd0ebe171586aaf8dd6b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 07:46:24 +0100 Subject: [PATCH 064/118] Add missing goals since tests are not failing anymore --- libbeat/Jenkinsfile.yml | 2 +- metricbeat/Jenkinsfile.yml | 2 +- x-pack/auditbeat/Jenkinsfile.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 700cb6ad2785..734b8814d765 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: build: - mage: "mage build" ## TODO test goal has been disabled since there are broken tests. + mage: "mage build test" crosscompile: make: "make -C libbeat crosscompile" stress-tests: diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index e73069b5644d..5cea25e0fed2 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: unitTest: - mage: "mage build" ## Disable unitTest since there are broken tests + mage: "mage build unitTest" goIntegTest: mage: "mage goIntegTest" withModule: true diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index f646fc6217e2..b0fd6ff198fa 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: build: - mage: "mage update build" # TODO: disable test until issues are fixed. + mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. macos: mage: "mage build unitTest" From 1624b54765e01ba2d56d61d2648d77f320416f33 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 07:59:16 +0100 Subject: [PATCH 065/118] Support testGCPFunctions --- x-pack/functionbeat/Jenkinsfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index ec62a2bb417e..d5d3c7de4f01 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,8 +13,8 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - build: ## TODO: missing testGCPFunctions - mage: "mage build test" + build: + mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 13fb5f004d5b2704725cf435c2ad5c2e35e589b0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 08:26:22 +0100 Subject: [PATCH 066/118] Set the project dependencies for the generators using the special token # --- generator/Jenkinsfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml index e2ebf371be4a..a2a9eb705db4 100644 --- a/generator/Jenkinsfile.yml +++ b/generator/Jenkinsfile.yml @@ -2,8 +2,8 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - "^generator/.*" - - "generator/common/beatgen" ## TODO getProjectDependencies - - "metricbeat/beater" ## TODO getProjectDependencies + - "#generator/common/beatgen" ## special token regarding the project dependency + - "#metricbeat/beater" ## special token regarding the project dependency - "@ci" ## special token regarding the changeset for the ci - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries From 5a22d2fee5ec817b7c3da867ee56bbef64bb6c66 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 08:51:29 +0100 Subject: [PATCH 067/118] Use eval gvm to avoid hardcoded PATHs --- .ci/Jenkinsfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 3d59867836da..8df86a15d8bd 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -19,8 +19,8 @@ pipeline { GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - PIPELINE_LOG_LEVEL = 'INFO' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + PIPELINE_LOG_LEVEL = 'INFO' RUNBLD_DISABLE_NOTIFICATIONS = 'true' TERRAFORM_VERSION = "0.12.24" XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' @@ -538,15 +538,12 @@ def normalise(String directory) { */ def getProjectDependencies(Map args = [:]) { def beatName = args.project - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" def output = "" - dir("${env.BASE_DIR}") { - withEnv([ - "HOME=${env.WORKSPACE}/${env.BASE_DIR}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - ]) { + withEnv(["HOME=${env.WORKSPACE}"]) { + dir("${env.BASE_DIR}") { output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ + eval \$(gvm "${GO_VERSION}") go list -deps ./${beatName} \ | grep 'elastic/beats' \ | sed -e "s#github.com/elastic/beats/v7/##g" \ From b5f99dedf274f23349c5f48f04731341cc126564 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 09:16:32 +0100 Subject: [PATCH 068/118] Fix linting issues with autopep8 --- .ci/scripts/generate_build_table.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py index b702019757c4..21fe5d671074 100755 --- a/.ci/scripts/generate_build_table.py +++ b/.ci/scripts/generate_build_table.py @@ -14,7 +14,7 @@ with open(os.path.join(root, file), 'r') as f: doc = yaml.load(f, Loader=yaml.FullLoader) module = root.replace(".{}".format(os.sep), '') - platforms = [ doc["platform"] ] + platforms = [doc["platform"]] when = "" if "branches" in doc["when"]: when = f"{when}/:palm_tree:" @@ -40,7 +40,8 @@ withModule = doc["stages"][stage]["withModule"] if "when" in doc["stages"][stage]: when = f"{when}/:star:" - print("| {} | {} | `{}` | {} | `{}` | {} |".format(module, stage, command, withModule, platforms, when)) + print("| {} | {} | `{}` | {} | `{}` | {} |".format( + module, stage, command, withModule, platforms, when)) print("> :palm_tree: -> Git Branch based") print("> :label: -> GitHub Pull Request Label based") From 269dee5b1a595cb415f44c712facb89fcdc52098 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 10:16:17 +0100 Subject: [PATCH 069/118] Folder is not required since the caller already configured it --- .ci/Jenkinsfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8df86a15d8bd..dd0c39f7d058 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -541,15 +541,14 @@ def getProjectDependencies(Map args = [:]) { def output = "" withEnv(["HOME=${env.WORKSPACE}"]) { - dir("${env.BASE_DIR}") { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - eval \$(gvm "${GO_VERSION}") - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) - } + output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """#!/usr/bin/env bash + set +x + eval \$(gvm "${GO_VERSION}") + go list -deps ./${beatName} \ + | grep 'elastic/beats' \ + | sed -e "s#github.com/elastic/beats/v7/##g" \ + | awk '{print "^" \$1 "/.*"}' + """) } return output?.split('\n').collect{ item -> item as String } } From 196e6bd9b69ba935ad4a97235ad15fac0e1ba871 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 10:20:48 +0100 Subject: [PATCH 070/118] Fix runbld issue --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index dd0c39f7d058..50bacf3ff854 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -401,9 +401,9 @@ def junitAndStore(Map args = [:]){ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${env.BASE_DIR}") { - sh(label: 'Prepare workspace context', - script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete') // Unstash the test reports stashedTestReports.each { k, v -> dir(k) { From 5785585181ded25b9088b7b22e747aaf6393356f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 10:35:30 +0100 Subject: [PATCH 071/118] Refactor a bit the cloud target --- .ci/Jenkinsfile | 32 +++++++++++++++---------------- x-pack/metricbeat/Jenkinsfile.yml | 2 ++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 50bacf3ff854..4e61e929f237 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -165,15 +165,13 @@ def runCommand(Map args = [:]) { k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label) } if(args?.content?.containsKey('cloud')) { - cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule) + cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs) } } def cloud(Map args = [:]) { node(args.label) { - stage("${args.context}-prepare-cloud-env"){ - startCloudTestEnv(name: args.directory, dirs: ['x-pack/metricbeat/module/aws']) - } + startCloudTestEnv(name: args.directory, dirs: args.dirs) } withCloudTestEnv() { try { @@ -469,19 +467,21 @@ def withCloudTestEnv(Closure body) { def startCloudTestEnv(Map args = [:]) { String name = normalise(args.name) def dirs = args.get('dirs',[]) - withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { - try { - for (folder in dirs) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(folder) + stage("${name}-prepare-cloud-env"){ + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + try { + for (folder in dirs) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ + terraformApply(folder) + } } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } } } @@ -489,12 +489,12 @@ def startCloudTestEnv(Map args = [:]) { /** * Tear down the terraform environments, by looking for all terraform states in directory * then it runs terraform destroy for each one. -* it uses terraform states previously stashed by startCloudTestEnv. +* It uses terraform states previously stashed by startCloudTestEnv. */ def terraformCleanup(Map args = [:]) { String name = normalise(args.name) String directory = args.dir - stage("Remove cloud scenarios in ${directory}"){ + stage("${name}-tear-down-cloud-env"){ withCloudTestEnv() { withBeatsEnv(archive: false, withModule: false) { unstash("terraform-${name}") diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 0bfc80317d31..57f524cd04fe 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -16,6 +16,8 @@ stages: build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## run the cloud tests for the given modules. + - "x-pack/metricbeat/module/aws" when: ## Aggregate when with the top-level one. parameters: - "awsCloudTests" From 4e27aa14e4b87c36dd23585b207ac47335ff26dd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 10:52:13 +0100 Subject: [PATCH 072/118] Refactor getProjectDependencies --- .ci/Jenkinsfile | 12 ++---------- .ci/scripts/get-vendor-dependencies.sh | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) create mode 100755 .ci/scripts/get-vendor-dependencies.sh diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4e61e929f237..c3fa5b977aca 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -537,18 +537,10 @@ def normalise(String directory) { * regex pattern. */ def getProjectDependencies(Map args = [:]) { - def beatName = args.project def output = "" - withEnv(["HOME=${env.WORKSPACE}"]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """#!/usr/bin/env bash - set +x - eval \$(gvm "${GO_VERSION}") - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) + output = sh(label: 'Get vendor dependency patterns', returnStdout: true, + script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") } return output?.split('\n').collect{ item -> item as String } } diff --git a/.ci/scripts/get-vendor-dependencies.sh b/.ci/scripts/get-vendor-dependencies.sh new file mode 100755 index 000000000000..5ad9d656fca1 --- /dev/null +++ b/.ci/scripts/get-vendor-dependencies.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Given the go module it will list all the dependencies that will be later on +# used by the CI to enable/disable specific stages as long as the changeset +# matches any of those patterns. +# +set -euo pipefail +GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"} +BEATS=${1:?"parameter missing."} +eval "$(gvm "${GO_VERSION}")" + +go list -deps ./"${BEATS}" \ +| grep 'elastic/beats' \ +| sort \ +| sed -e "s#github.com/elastic/beats/v7/##g" \ +| awk '{print "^" $1 "/.*"}' From 414a8f0142a4d79c0ed73093777eaa3fb937be36 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 10:57:40 +0100 Subject: [PATCH 073/118] Use ubuntu-18 and disable windows-2016 --- auditbeat/Jenkinsfile.yml | 3 +-- x-pack/auditbeat/Jenkinsfile.yml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 28f57a165b70..6cbb0ca42593 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -11,7 +11,7 @@ when: parameters: ## when parameter was selected in the UI. - "auditbeat" tags: true ## for all the tags -platform: "linux && ubuntu-16" ## default label for all the stages +platform: "linux && ubuntu-18" ## default label for all the stages stages: build: mage: "mage build test" @@ -32,4 +32,3 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - - "windows-2016" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index b0fd6ff198fa..82867feae559 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -31,4 +31,3 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - - "windows-2016" From 9e97630a0971ad103992a77ad60c858a8ab78b18 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 11:25:36 +0100 Subject: [PATCH 074/118] [runbld] debug what's going on --- .ci/Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index c3fa5b977aca..fda4c340afa5 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -410,6 +410,8 @@ def runbld() { } sh(label: 'Process JUnit reports with runbld', script: '''\ + ## for debugging purposes + find . -name "TEST-*.xml" cat >./runbld-script < Date: Fri, 31 Jul 2020 11:51:12 +0100 Subject: [PATCH 075/118] Avoid to error if no Go files in --- .ci/scripts/get-vendor-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/get-vendor-dependencies.sh b/.ci/scripts/get-vendor-dependencies.sh index 5ad9d656fca1..e002a208b760 100755 --- a/.ci/scripts/get-vendor-dependencies.sh +++ b/.ci/scripts/get-vendor-dependencies.sh @@ -4,7 +4,7 @@ # used by the CI to enable/disable specific stages as long as the changeset # matches any of those patterns. # -set -euo pipefail + GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"} BEATS=${1:?"parameter missing."} eval "$(gvm "${GO_VERSION}")" From 7339f12d191c3a9d101fa4003c222473664f00d1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 12:13:49 +0100 Subject: [PATCH 076/118] [ci] use previous approach to support docker for mac issues --- .ci/Jenkinsfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index fda4c340afa5..b343b6a6afb8 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -276,7 +276,7 @@ def withBeatsEnv(Map args = [:], Closure body) { "TEST_COVERAGE=true", "TEST_TAGS=${env.TEST_TAGS},oracle" ]) { - if(isInstalled(tool: 'docker', flag: '--version')) { + if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { @@ -608,3 +608,12 @@ def dumpVariables(){ ### END ENV DUMP ### """ } + +def isDockerInstalled(){ + if (isUnix()) { + // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) { + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + } else { + return false + } +} From 9727b52b6f69bd2c4e4c20d66f56c384ea01913d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 16:51:19 +0100 Subject: [PATCH 077/118] [POC] Prepare context for the skip/draft PRs --- Jenkinsfile.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 95780826d6b1..2f720bf055b2 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -36,4 +36,12 @@ changeset: - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" - - "^x-pack/libbeat/.*" \ No newline at end of file + - "^x-pack/libbeat/.*" + +## Proposal +## TBC: This will allow to configure what to do based on the PR configuration +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if there is a GitHub label match + - "skip-ci" + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. From a492e6b34c6b74caf734658823e23d25ffd32fcf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jul 2020 16:51:53 +0100 Subject: [PATCH 078/118] [RUNBLD] force JOB-NAME to match --- .ci/Jenkinsfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index b343b6a6afb8..e787825e768f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -399,6 +399,10 @@ def junitAndStore(Map args = [:]){ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { + def jobName = "elastic+beats" + if (isPR()) { + jobName = "elastic+beats+pull-request" + } deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${env.BASE_DIR}") { @@ -409,14 +413,14 @@ def runbld() { } } sh(label: 'Process JUnit reports with runbld', - script: '''\ + script: """\ ## for debugging purposes find . -name "TEST-*.xml" cat >./runbld-script < Date: Thu, 6 Aug 2020 07:58:40 +0100 Subject: [PATCH 079/118] [CI] runbld support STAGE_NAME env variable is not anymore available when using the parallel step, for such it required to pass the id that contains the unique id for the parallel sttages created on the fly, and also using a different naming for the runbld project to avoid any match with the 'ansible/roles/runbld/templates/runbld.conf.j2' --- .ci/Jenkinsfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index e787825e768f..73062fae8b93 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -156,16 +156,16 @@ def generateStages(Map args = [:]) { def runCommand(Map args = [:]) { def withModule = args.content.get('withModule', false) if(args?.content?.containsKey('make')) { - target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false) + target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) } if(args?.content?.containsKey('mage')) { - target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true) + target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) } if(args?.content?.containsKey('k8sTest')) { - k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label) + k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) } if(args?.content?.containsKey('cloud')) { - cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs) + cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) } } @@ -175,7 +175,7 @@ def cloud(Map args = [:]) { } withCloudTestEnv() { try { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true) + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) } finally { terraformCleanup(name: args.directory, dir: args.directory) } @@ -220,7 +220,7 @@ def target(Map args = [:]) { def isMage = args.get('isMage', false) node(args.label) { withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { dumpVariables() // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. @@ -294,7 +294,7 @@ def withBeatsEnv(Map args = [:], Closure body) { body() } finally { if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts) + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) } // Tear down the setup for the permamnent workers. catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { @@ -384,10 +384,10 @@ def archiveTestOutput(Map args = [:]) { * This method wraps the junit built-in step to archive the test reports that gonna be populated later on * with the runbld post build step. */ -def junitAndStore(Map args = [:]){ +def junitAndStore(Map args = [:]) { junit(args) - // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds - def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" + // args.id could be null in some cases, so let's use the currentmilliseconds + def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) stashedTestReports[stageName] = stageName } @@ -399,9 +399,9 @@ def junitAndStore(Map args = [:]){ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { - def jobName = "elastic+beats" + def jobName = "elastic+beats-new" if (isPR()) { - jobName = "elastic+beats+pull-request" + jobName = "elastic+beats-new+pull-request" } deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") From f701834ea921bcfd53a8bf8571639069c96ca417 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 6 Aug 2020 10:32:20 +0100 Subject: [PATCH 080/118] [CI] force cwd and pass the id args --- .ci/Jenkinsfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 73062fae8b93..7ed8b513304f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -366,7 +366,7 @@ def archiveTestOutput(Map args = [:]) { } cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { @@ -399,13 +399,16 @@ def junitAndStore(Map args = [:]) { def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { - def jobName = "elastic+beats-new" + def jobName = 'elastic+beats' + // TODO: change ansible/roles/runbld/templates/runbld.conf.j2 + def where = '' if (isPR()) { - jobName = "elastic+beats-new+pull-request" + jobName = 'elastic+beats+pull-request' + where = env.BASE_DIR } deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${env.BASE_DIR}") { + dir("${where}") { // Unstash the test reports stashedTestReports.each { k, v -> dir(k) { From 0395eb896367a2dc4778c2fdde07b8206b1a32d8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 6 Aug 2020 13:53:38 +0100 Subject: [PATCH 081/118] fixes The most common cause is that Jenkins and runbld are configured with different working directories --- .ci/Jenkinsfile | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 7ed8b513304f..014704f6db55 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -400,31 +400,26 @@ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { def jobName = 'elastic+beats' - // TODO: change ansible/roles/runbld/templates/runbld.conf.j2 - def where = '' if (isPR()) { jobName = 'elastic+beats+pull-request' - where = env.BASE_DIR } deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${where}") { - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) } - sh(label: 'Process JUnit reports with runbld', - script: """\ - ## for debugging purposes - find . -name "TEST-*.xml" - cat >./runbld-script <./runbld-script < Date: Thu, 6 Aug 2020 15:45:43 +0100 Subject: [PATCH 082/118] Tests need to be stored in the workspace --- .ci/Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 014704f6db55..f0c6335591ab 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -394,21 +394,21 @@ def junitAndStore(Map args = [:]) { /** * This method populates the test output using the runbld approach. For such it requires the -* global variable stashedTestReports. +* global variable stashedTestReports. +* TODO: should be moved to the shared library */ def runbld() { catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { if (stashedTestReports) { - def jobName = 'elastic+beats' - if (isPR()) { - jobName = 'elastic+beats+pull-request' - } + def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) + dir("${env.BASE_DIR}") { + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) + } } } sh(label: 'Process JUnit reports with runbld', From d431b02df228b2aec4047455ac28dfad5d9aec5e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 12 Aug 2020 16:26:45 +0200 Subject: [PATCH 083/118] Test AWS --- Jenkinsfile.yml | 34 +++++++++++++++---------------- x-pack/metricbeat/Jenkinsfile.yml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 2f720bf055b2..bc881f08c754 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,22 +1,22 @@ projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "generator" - - "heartbeat" - - "journalbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/elastic-agent" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/libbeat" + #- "auditbeat" + #- "deploy/kubernetes" + #- "filebeat" + #- "generator" + #- "heartbeat" + #- "journalbeat" + #- "libbeat" + #- "metricbeat" + #- "packetbeat" + #- "winlogbeat" + #- "x-pack/auditbeat" + #- "x-pack/dockerlogbeat" + #- "x-pack/elastic-agent" + #- "x-pack/filebeat" + #- "x-pack/functionbeat" + #- "x-pack/libbeat" - "x-pack/metricbeat" - - "x-pack/winlogbeat" + #- "x-pack/winlogbeat" ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 57f524cd04fe..6ee23f7e2736 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -21,7 +21,7 @@ stages: when: ## Aggregate when with the top-level one. parameters: - "awsCloudTests" - - "runAllCloudTests" + - "allCloudTests" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 993f336c53b9291049ac8c29704c516411cfa675 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 3 Sep 2020 09:08:43 +0100 Subject: [PATCH 084/118] Revert "Test AWS" This reverts commit d431b02df228b2aec4047455ac28dfad5d9aec5e. --- Jenkinsfile.yml | 34 +++++++++++++++---------------- x-pack/metricbeat/Jenkinsfile.yml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index bc881f08c754..2f720bf055b2 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,22 +1,22 @@ projects: - #- "auditbeat" - #- "deploy/kubernetes" - #- "filebeat" - #- "generator" - #- "heartbeat" - #- "journalbeat" - #- "libbeat" - #- "metricbeat" - #- "packetbeat" - #- "winlogbeat" - #- "x-pack/auditbeat" - #- "x-pack/dockerlogbeat" - #- "x-pack/elastic-agent" - #- "x-pack/filebeat" - #- "x-pack/functionbeat" - #- "x-pack/libbeat" + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "generator" + - "heartbeat" + - "journalbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/elastic-agent" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/libbeat" - "x-pack/metricbeat" - #- "x-pack/winlogbeat" + - "x-pack/winlogbeat" ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 6ee23f7e2736..57f524cd04fe 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -21,7 +21,7 @@ stages: when: ## Aggregate when with the top-level one. parameters: - "awsCloudTests" - - "allCloudTests" + - "runAllCloudTests" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 46c3ad9cfa8c741e000a3b4bc576ee81fcf47927 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 3 Sep 2020 09:09:39 +0100 Subject: [PATCH 085/118] [CI] Fix params name --- x-pack/metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 57f524cd04fe..6ee23f7e2736 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -21,7 +21,7 @@ stages: when: ## Aggregate when with the top-level one. parameters: - "awsCloudTests" - - "runAllCloudTests" + - "allCloudTests" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 9ce54de48a8b5fa4ce7def7ceeeb12f32fb0d030 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 3 Sep 2020 09:13:24 +0100 Subject: [PATCH 086/118] Disable cloud as long as https://github.com/elastic/beats/issues/20951 is not fixed --- x-pack/metricbeat/Jenkinsfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 6ee23f7e2736..e4d01b5744dd 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -22,6 +22,7 @@ stages: parameters: - "awsCloudTests" - "allCloudTests" + disabled: true ## See https://github.com/elastic/beats/issues/20951 macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From a3da23200b60535bf14ed226468def60f98a5003 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 3 Sep 2020 14:27:47 +0100 Subject: [PATCH 087/118] Fix wound up catching method mismatch with this.& by using a Class- --- .ci/Jenkinsfile | 112 ++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index f0c6335591ab..72a081a74d22 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -134,8 +134,8 @@ def generateStages(Map args = [:]) { if (fileExists(fileName)) { def content = readYaml(file: fileName) // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. - if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: this.&getProjectDependencies)) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: this.&runCommand) + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) } } else { log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") @@ -143,32 +143,6 @@ def generateStages(Map args = [:]) { return mapParallelStages } -/** -* This method is the one used for running the parallel stages, therefore -* its arguments are passed by the beatsStages step. -* -* What parameters/arguments are supported: -* - label -> the worker labels -* - project -> the name of the project that should match with the folder name. -* - content -> the specific stage data in the /Jenkinsfile.yml -* - context -> the name of the stage, normally -(-)? -*/ -def runCommand(Map args = [:]) { - def withModule = args.content.get('withModule', false) - if(args?.content?.containsKey('make')) { - target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) - } - if(args?.content?.containsKey('mage')) { - target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) - } - if(args?.content?.containsKey('k8sTest')) { - k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) - } - if(args?.content?.containsKey('cloud')) { - cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) - } -} - def cloud(Map args = [:]) { node(args.label) { startCloudTestEnv(name: args.directory, dirs: args.dirs) @@ -490,6 +464,16 @@ def startCloudTestEnv(Map args = [:]) { } } +/** +* Run terraform in the given directory +*/ +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} + /** * Tear down the terraform environments, by looking for all terraform states in directory * then it runs terraform destroy for each one. @@ -519,16 +503,6 @@ def terraformInit(String directory) { } } -/** -* Run terraform in the given directory -*/ -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } -} - /** * Replace the slashes in the directory in case there are nested folders. */ @@ -536,19 +510,6 @@ def normalise(String directory) { return directory.replaceAll("[\\W]|_",'-') } -/** -* This method fetchs the dependencies of a Go module for such it transforms them in a -* regex pattern. -*/ -def getProjectDependencies(Map args = [:]) { - def output = "" - withEnv(["HOME=${env.WORKSPACE}"]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, - script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") - } - return output?.split('\n').collect{ item -> item as String } -} - /** * For debugging purposes. */ @@ -619,3 +580,52 @@ def isDockerInstalled(){ return false } } + +/** +* This class is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +* +* What parameters/arguments are supported: +* - label -> the worker labels +* - project -> the name of the project that should match with the folder name. +* - content -> the specific stage data in the /Jenkinsfile.yml +* - context -> the name of the stage, normally -(-)? +*/ +class RunCommand extends co.elastic.beats.BeatsFunction { + public RunCommand(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def withModule = args.content.get('withModule', false) + if(args?.content?.containsKey('make')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('mage')) { + steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) + } + if(args?.content?.containsKey('k8sTest')) { + steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) + } + if(args?.content?.containsKey('cloud')) { + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) + } + } +} + +/** +* This class retrieves the dependencies of a Go module for such it transforms them in a +* regex pattern. +*/ +class GetProjectDependencies extends co.elastic.beats.BeatsFunction { + public GetProjectDependencies(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def output = "" + steps.withEnv(["HOME=${steps.env.WORKSPACE}"]) { + output = steps.sh(label: 'Get vendor dependency patterns', returnStdout: true, + script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") + } + return output?.split('\n').collect{ item -> item as String } + } +} \ No newline at end of file From b08c40a62d2478a06948f68e0b153323132c196d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 3 Sep 2020 16:14:50 +0100 Subject: [PATCH 088/118] Enable cloud stages --- x-pack/metricbeat/Jenkinsfile.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index e4d01b5744dd..6ee23f7e2736 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -22,7 +22,6 @@ stages: parameters: - "awsCloudTests" - "allCloudTests" - disabled: true ## See https://github.com/elastic/beats/issues/20951 macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From bba848dd8ee89d37a6716190a32f95309b1e5467 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 14 Sep 2020 13:53:32 +0100 Subject: [PATCH 089/118] Apply suggestions from code review Co-authored-by: cachedout --- auditbeat/Jenkinsfile.yml | 2 +- deploy/kubernetes/Jenkinsfile.yml | 4 ++-- filebeat/Jenkinsfile.yml | 2 +- generator/Jenkinsfile.yml | 2 +- heartbeat/Jenkinsfile.yml | 2 +- journalbeat/Jenkinsfile.yml | 2 +- libbeat/Jenkinsfile.yml | 2 +- metricbeat/Jenkinsfile.yml | 2 +- packetbeat/Jenkinsfile.yml | 2 +- winlogbeat/Jenkinsfile.yml | 2 +- x-pack/auditbeat/Jenkinsfile.yml | 2 +- x-pack/dockerlogbeat/Jenkinsfile.yml | 2 +- x-pack/elastic-agent/Jenkinsfile.yml | 2 +- x-pack/filebeat/Jenkinsfile.yml | 2 +- x-pack/functionbeat/Jenkinsfile.yml | 2 +- x-pack/libbeat/Jenkinsfile.yml | 2 +- x-pack/metricbeat/Jenkinsfile.yml | 2 +- x-pack/packetbeat/Jenkinsfile.yml | 2 +- x-pack/winlogbeat/Jenkinsfile.yml | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 6cbb0ca42593..873e2c319f3e 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test auditbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "auditbeat" parameters: ## when parameter was selected in the UI. - "auditbeat" diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 532523cc90f9..452771edfb5d 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -4,7 +4,7 @@ when: - "^deploy/kubernetes/.*" comments: ## when PR comment contains any of those entries - "/test deploy/kubernetes" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "kubernetes" parameters: ## when parameter was selected in the UI. - "kubernetes" @@ -12,4 +12,4 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: k8sTest: - k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" \ No newline at end of file + k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 5c86b58f8229..45b032accfb6 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test filebeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "filebeat" parameters: ## when parameter was selected in the UI. - "filebeat" diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml index a2a9eb705db4..071d24858bbe 100644 --- a/generator/Jenkinsfile.yml +++ b/generator/Jenkinsfile.yml @@ -8,7 +8,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test generator" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "generator" parameters: ## when parameter was selected in the UI. - "generator" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 5bf8677635b7..b8668715c3c4 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test heartbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "heartbeat" parameters: ## when parameter was selected in the UI. - "heartbeat" diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 8f4877576f10..12bb63f4cc64 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test journalbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "journalbeat" parameters: ## when parameter was selected in the UI. - "journalbeat" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 734b8814d765..64a43269b13d 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -5,7 +5,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test libbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "libbeat" parameters: ## when parameter was selected in the UI. - "libbeat" diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 5cea25e0fed2..1219a27af77f 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test metricbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "metricbeat" parameters: ## when parameter was selected in the UI. - "metricbeat" diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 8a2e125aaaa5..416e69a203ba 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test packetbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "packetbeat" parameters: ## when parameter was selected in the UI. - "packetbeat" diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index e2a185ffffa0..74eb55586d0d 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@oss" ## special token regarding the changeset for the oss comments: ## when PR comment contains any of those entries - "/test winlogbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "winlogbeat" parameters: ## when parameter was selected in the UI. - "winlogbeat" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 82867feae559..86f0832d3f27 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test auditbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "auditbeat" parameters: ## when parameter was selected in the UI. - "auditbeat" diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml index 5a4dfe68908f..703bb3d66a9d 100644 --- a/x-pack/dockerlogbeat/Jenkinsfile.yml +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/dockerlogbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-dockerlogbeat" parameters: ## when parameter was selected in the UI. - "x-pack-dockerlogbeat" diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 91fc0e847090..8f99e11da3c7 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/elastic-agent" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-elastic-agent" parameters: ## when parameter was selected in the UI. - "x-pack-elastic-agent" diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 3ccd0c23b757..d3d5e6d862eb 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/filebeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-filebeat" parameters: ## when parameter was selected in the UI. - "x-pack-filebeat" diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index d5d3c7de4f01..f3428ae7cc83 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/functionbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-functionbeat" parameters: ## when parameter was selected in the UI. - "x-pack-functionbeat" diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index a7c217d4bc6d..87019f071a09 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/libbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-libbeat" parameters: ## when parameter was selected in the UI. - "x-pack-libbeat" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 6ee23f7e2736..365f1e116b4f 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/metricbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-metricbeat" parameters: ## when parameter was selected in the UI. - "x-pack-metricbeat" diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index 01dfe0e11c60..8496265e0ac2 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/winlogbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-winlogbeat" parameters: ## when parameter was selected in the UI. - "x-pack-winlogbeat" diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index d6dc55c04fb7..396d1f03a7ca 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -6,7 +6,7 @@ when: - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - "/test x-pack/winlogbeat" - labels: ## when PR labels matche any of those entries + labels: ## when PR labels matches any of those entries - "x-pack-winlogbeat" parameters: ## when parameter was selected in the UI. - "x-pack-winlogbeat" From bb7cd784c36511381970ca4b213ddfb6d453a6f0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 14 Sep 2020 16:09:47 +0100 Subject: [PATCH 090/118] Update .ci/Jenkinsfile --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 72a081a74d22..c8ecc961a1da 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -1,6 +1,6 @@ #!/usr/bin/env groovy -@Library('apm@feature/2.0-beats') _ +@Library('apm@current') _ import groovy.transform.Field /** @@ -628,4 +628,4 @@ class GetProjectDependencies extends co.elastic.beats.BeatsFunction { } return output?.split('\n').collect{ item -> item as String } } -} \ No newline at end of file +} From 61621c519ea6e2555a4a3c297bbac3f0352bc882 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 14 Sep 2020 16:12:07 +0100 Subject: [PATCH 091/118] Update .ci/Jenkinsfile Co-authored-by: cachedout --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index c8ecc961a1da..9f297f6c5ab1 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -312,7 +312,7 @@ def installTools() { * the changeset affects a specific module. * * For such, it's required to look for changes under the module folder and exclude anything else -* such as ascidoc and png files. +* such as asciidoc and png files. */ def getCommonModuleInTheChangeSet(String directory) { // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') From 34955f2c09cb487cb0e02eb424f02210180e188b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 15 Sep 2020 11:54:13 +0100 Subject: [PATCH 092/118] As commented in the review --- .ci/Jenkinsfile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9f297f6c5ab1..a11de7c36afa 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -68,7 +68,16 @@ pipeline { environment { GOFLAGS = '-mod=readonly' } + steps { + target(context: 'Lint', target: 'make check') + } + } + stage('Build&Test') { + options { skipDefaultCheckout() } when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter anyOf { not { changeRequest() } // If no PR allOf { // If PR and no docs changes @@ -78,14 +87,6 @@ pipeline { expression { return params.runAllStages } // If UI forced } } - steps { - whenFalse(true) { // TODO: disable for the time being. - target(context: "Lint", target: 'make check') - } - } - } - stage('Build&Test') { - options { skipDefaultCheckout() } steps { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") From 987d8802ac26edf4be6f5755224e6118b23aa018 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 15 Sep 2020 11:55:56 +0100 Subject: [PATCH 093/118] As commented in the review Let's run always and leave the logic to setup the cloud to the function itself this will support two scenarios: when cloud parameters wills etup the environment, and run the tests, or run the tests only --- x-pack/metricbeat/Jenkinsfile.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 365f1e116b4f..2448d43d85b1 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -18,10 +18,6 @@ stages: withModule: true ## run the ITs only if the changeset affects a specific module. dirs: ## run the cloud tests for the given modules. - "x-pack/metricbeat/module/aws" - when: ## Aggregate when with the top-level one. - parameters: - - "awsCloudTests" - - "allCloudTests" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 8374f56180c7bf33019b3edc4a3f5e2ea7d43116 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 16 Sep 2020 17:49:50 +0100 Subject: [PATCH 094/118] Enable target without a node --- .ci/Jenkinsfile | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a11de7c36afa..079cd603cc5e 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -69,7 +69,8 @@ pipeline { GOFLAGS = '-mod=readonly' } steps { - target(context: 'Lint', target: 'make check') + // Use targetCommand to reuse the top level worker + targetCommand(context: 'Lint', target: 'make check') } } stage('Build&Test') { @@ -187,23 +188,37 @@ def k8sTest(Map args = [:]) { * - make -C then the dir(location) is not required, aka by disaling isMage: false * - mage then the dir(location) is required, aka by enabling isMage: true. */ +def targetCommand(Map args = [:]) { + def context = args.context + def command = args.command + def directory = args.directory + def withModule = args.withModule + def isMage = args.isMage + def id = args.id + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: id) { + dumpVariables() + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + cmd(label: "${command}", script: "${command}") + } + } + } +} + +/** +* This method runs the targetCommand in a given worker. +*/ def target(Map args = [:]) { def context = args.context def command = args.command + def id = args.id def directory = args.get('directory', '') def withModule = args.get('withModule', false) def isMage = args.get('isMage', false) node(args.label) { - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { - dumpVariables() - // make commands use -C while mage commands require the dir(folder) - // let's support this scenario with the location variable. - dir(isMage ? directory : '') { - cmd(label: "${command}", script: "${command}") - } - } - } + targetCommand(context: context, command: command, directory: directory, withModule: withModule, isMage: isMage, id: id) } } From 5bda33f811ef1aef099c4e2f01b53cfa9ec8c9a3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 17 Sep 2020 07:27:42 +0100 Subject: [PATCH 095/118] Revert "Enable target without a node" This reverts commit 8374f56180c7bf33019b3edc4a3f5e2ea7d43116. --- .ci/Jenkinsfile | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 079cd603cc5e..a11de7c36afa 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -69,8 +69,7 @@ pipeline { GOFLAGS = '-mod=readonly' } steps { - // Use targetCommand to reuse the top level worker - targetCommand(context: 'Lint', target: 'make check') + target(context: 'Lint', target: 'make check') } } stage('Build&Test') { @@ -188,37 +187,23 @@ def k8sTest(Map args = [:]) { * - make -C then the dir(location) is not required, aka by disaling isMage: false * - mage then the dir(location) is required, aka by enabling isMage: true. */ -def targetCommand(Map args = [:]) { - def context = args.context - def command = args.command - def directory = args.directory - def withModule = args.withModule - def isMage = args.isMage - def id = args.id - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: id) { - dumpVariables() - // make commands use -C while mage commands require the dir(folder) - // let's support this scenario with the location variable. - dir(isMage ? directory : '') { - cmd(label: "${command}", script: "${command}") - } - } - } -} - -/** -* This method runs the targetCommand in a given worker. -*/ def target(Map args = [:]) { def context = args.context def command = args.command - def id = args.id def directory = args.get('directory', '') def withModule = args.get('withModule', false) def isMage = args.get('isMage', false) node(args.label) { - targetCommand(context: context, command: command, directory: directory, withModule: withModule, isMage: isMage, id: id) + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + dumpVariables() + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + cmd(label: "${command}", script: "${command}") + } + } + } } } From b4a1fafaa9e008aa86bd32fcaf0d6a1906d09c17 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 17 Sep 2020 07:34:38 +0100 Subject: [PATCH 096/118] Simplify the lint stage compared to the previous approach --- .ci/Jenkinsfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a11de7c36afa..e7f91ef95e5e 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -69,7 +69,12 @@ pipeline { GOFLAGS = '-mod=readonly' } steps { - target(context: 'Lint', target: 'make check') + withGithubNotify(context: 'Lint') { + withBeatsEnv(archive: true) { + dumpVariables() + cmd(label: 'make check', script: 'make check') + } + } } } stage('Build&Test') { From d7339674fc86ee48be56de1482698af34f726d5c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 09:27:49 +0100 Subject: [PATCH 097/118] [CI] add arm build for auditbeat --- auditbeat/Jenkinsfile.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 873e2c319f3e..39218aed3948 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -17,6 +17,17 @@ stages: mage: "mage build test" crosscompile: make: "make -C auditbeat crosscompile" + arm: + mage: "mage build" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 6e9f8bceac94d663247830652d6e8e76a932e73b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 09:28:01 +0100 Subject: [PATCH 098/118] chore: for testing purposes --- Jenkinsfile.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 2f720bf055b2..3616b1e3a5cd 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,22 +1,22 @@ projects: - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "generator" - - "heartbeat" - - "journalbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/elastic-agent" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/winlogbeat" + #- "deploy/kubernetes" + #- "filebeat" + #- "generator" + #- "heartbeat" + #- "journalbeat" + #- "libbeat" + #- "metricbeat" + #- "packetbeat" + #- "winlogbeat" + #- "x-pack/auditbeat" + #- "x-pack/dockerlogbeat" + #- "x-pack/elastic-agent" + #- "x-pack/filebeat" + #- "x-pack/functionbeat" + #- "x-pack/libbeat" + #- "x-pack/metricbeat" + #- "x-pack/winlogbeat" ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. From 8d33c8e295f549ffc83342134908e379616a3430 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 11:25:15 +0100 Subject: [PATCH 099/118] Add arm parameter --- .ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index e7f91ef95e5e..ead024369a21 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -44,6 +44,7 @@ pipeline { string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.') } stages { stage('Checkout') { From 87673f06ef69d3203598f993a671f106f9871f27 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 12:01:38 +0100 Subject: [PATCH 100/118] chore: disable for testing purposes --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index ead024369a21..3754d7b7b078 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -73,7 +73,7 @@ pipeline { withGithubNotify(context: 'Lint') { withBeatsEnv(archive: true) { dumpVariables() - cmd(label: 'make check', script: 'make check') + //cmd(label: 'make check', script: 'make check') } } } From 3b07830f98892a9783aff482787a65a37af86b3c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 12:29:47 +0100 Subject: [PATCH 101/118] Support ARM arch --- .ci/scripts/install-go.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 49e12c7a18d3..59f56445a802 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -5,7 +5,8 @@ MSG="environment variable missing" GO_VERSION=${GO_VERSION:?$MSG} PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} HOME=${HOME:?$MSG} -ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') GVM_CMD="${HOME}/bin/gvm" if command -v go @@ -19,10 +20,20 @@ then fi fi +if [ "${ARCH}" == "aarch64" ] ; then + GVM_ARCH_SUFFIX=arm64 +elif [ "${ARCH}" == "x86_64" ] ; then + GVM_ARCH_SUFFIX=amd64 +elif [ "${ARCH}" == "i686" ] ; then + GVM_ARCH_SUFFIX=386 +else + GVM_ARCH_SUFFIX=arm +fi + echo "UNMET DEP: Installing Go" mkdir -p "${HOME}/bin" -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}" chmod +x "${GVM_CMD}" gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} From 009e958a2d000f707d0ded442ffca2af9d9468da Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 12:42:00 +0100 Subject: [PATCH 102/118] Support ARM arch in the pipeline --- .ci/Jenkinsfile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 3754d7b7b078..52cd99c02ac0 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -16,7 +16,6 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' @@ -221,10 +220,17 @@ def withBeatsEnv(Map args = [:], Closure body) { def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def goRoot, path, magefile, pythonEnv, testResults, artifacts + def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags if(isUnix()) { - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + if (isArm() && is64arm()) { + // TODO: nodeOS() should support ARM + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64" + gox_flags = '-arch arm' + } else { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + gox_flags = '-arch amd64' + } path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" @@ -238,6 +244,7 @@ def withBeatsEnv(Map args = [:], Closure body) { magefile = "${env.WORKSPACE}\\.magefile" testResults = "**\\build\\TEST*.xml" artifacts = "**\\build\\TEST*.out" + gox_flags = '-arch amd64' } deleteDir() @@ -255,7 +262,8 @@ def withBeatsEnv(Map args = [:], Closure body) { "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle" + "TEST_TAGS=${env.TEST_TAGS},oracle", + "GOX_FLAGS=${gox_flags}" ]) { if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") @@ -354,7 +362,9 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() + // TODO: nodeOS() should support ARM + def os_suffix = isArm() ? 'linux' : nodeOS() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix tar(file: "${name}.tgz", archive: true, dir: folder) } } From 8a6f2d2e730816105816f156eb77c9c73a1e89f1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 12:42:51 +0100 Subject: [PATCH 103/118] chore: disable for testing purposes --- auditbeat/Jenkinsfile.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 39218aed3948..b1e1266cde87 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,33 +13,7 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - build: - mage: "mage build test" - crosscompile: - make: "make -C auditbeat crosscompile" arm: mage: "mage build" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. - comments: - - "/test auditbeat for arm" - labels: - - "arm" - parameters: - - "armTest" - macos: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "macosx" - when: ## Aggregate when with the top-level one. - comments: - - "/test auditbeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - windows: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2019" From f7163daf2d4c3fc3c66a9f8d674f4da9cd4fca1a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 14:51:16 +0100 Subject: [PATCH 104/118] chore: test changes in the library --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 52cd99c02ac0..80971b24a283 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -1,6 +1,6 @@ #!/usr/bin/env groovy -@Library('apm@current') _ +@Library('apm@feature/support-arm') _ import groovy.transform.Field /** From b3f75a889cc090f5beae33fdd3d254278545ef23 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:10:24 +0100 Subject: [PATCH 105/118] Support unitTest in ARM --- auditbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index b1e1266cde87..78dfdd6bf2bd 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -14,6 +14,6 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" From bb014cad945ae50ea9ed59d3d5b17a3aa2f1ff08 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:26:41 +0100 Subject: [PATCH 106/118] Support test in ARM --- auditbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 78dfdd6bf2bd..e42d4cd20224 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -14,6 +14,6 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build unitTest" + mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" From 5b51b1599ebeb45aaeac21503414b8e037078796 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:32:35 +0100 Subject: [PATCH 107/118] Revert "chore: for testing purposes" This reverts commit 6e9f8bceac94d663247830652d6e8e76a932e73b. --- Jenkinsfile.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 3616b1e3a5cd..2f720bf055b2 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,22 +1,22 @@ projects: - "auditbeat" - #- "deploy/kubernetes" - #- "filebeat" - #- "generator" - #- "heartbeat" - #- "journalbeat" - #- "libbeat" - #- "metricbeat" - #- "packetbeat" - #- "winlogbeat" - #- "x-pack/auditbeat" - #- "x-pack/dockerlogbeat" - #- "x-pack/elastic-agent" - #- "x-pack/filebeat" - #- "x-pack/functionbeat" - #- "x-pack/libbeat" - #- "x-pack/metricbeat" - #- "x-pack/winlogbeat" + - "deploy/kubernetes" + - "filebeat" + - "generator" + - "heartbeat" + - "journalbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/elastic-agent" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/winlogbeat" ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. From 8bb0bb9200e26dd6d49d759fe450d264560b3388 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:38:13 +0100 Subject: [PATCH 108/118] Add remaining beats --- filebeat/Jenkinsfile.yml | 4 ++++ heartbeat/Jenkinsfile.yml | 4 ++++ journalbeat/Jenkinsfile.yml | 6 +++++- libbeat/Jenkinsfile.yml | 4 ++++ metricbeat/Jenkinsfile.yml | 4 ++++ packetbeat/Jenkinsfile.yml | 4 ++++ x-pack/auditbeat/Jenkinsfile.yml | 4 ++++ x-pack/elastic-agent/Jenkinsfile.yml | 4 ++++ x-pack/filebeat/Jenkinsfile.yml | 4 ++++ x-pack/functionbeat/Jenkinsfile.yml | 4 ++++ x-pack/libbeat/Jenkinsfile.yml | 4 ++++ x-pack/metricbeat/Jenkinsfile.yml | 4 ++++ 12 files changed, 49 insertions(+), 1 deletion(-) diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 45b032accfb6..4e4c9e9235d3 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index b8668715c3c4..681691810d65 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" macos: diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 12bb63f4cc64..e162af761ef6 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,5 +13,9 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - unitTest: + arm: mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + unitTest: + mage: "mage build unitTest" \ No newline at end of file diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 64a43269b13d..d4e2e668640d 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,6 +12,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" crosscompile: diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 1219a27af77f..668aee9b2053 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" unitTest: mage: "mage build unitTest" goIntegTest: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 416e69a203ba..f5e46200d240 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" macos: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 86f0832d3f27..690a7aff42aa 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 8f99e11da3c7..80a43601ad77 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" macos: diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index d3d5e6d862eb..77b337edbfa6 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index f3428ae7cc83..309fbcd195d6 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" macos: diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 87019f071a09..28ec5c807c10 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,5 +13,9 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "arm" build: mage: "mage build test" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 2448d43d85b1..84d1ae382e68 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,6 +13,10 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. From fb1171f860bffa92d16740e56def03101ace2918 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:38:52 +0100 Subject: [PATCH 109/118] Revert "chore: disable for testing purposes" This reverts commit 8a6f2d2e730816105816f156eb77c9c73a1e89f1. --- auditbeat/Jenkinsfile.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index e42d4cd20224..2a98cb03e52d 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,7 +13,33 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + build: + mage: "mage build test" + crosscompile: + make: "make -C auditbeat crosscompile" arm: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" From 2e425bdc2bf36658d4b8db4ff0b178111cb33bd8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:42:12 +0100 Subject: [PATCH 110/118] Add when conditions --- filebeat/Jenkinsfile.yml | 7 +++++++ heartbeat/Jenkinsfile.yml | 7 +++++++ journalbeat/Jenkinsfile.yml | 7 +++++++ libbeat/Jenkinsfile.yml | 7 +++++++ metricbeat/Jenkinsfile.yml | 7 +++++++ packetbeat/Jenkinsfile.yml | 7 +++++++ x-pack/auditbeat/Jenkinsfile.yml | 7 +++++++ x-pack/elastic-agent/Jenkinsfile.yml | 7 +++++++ x-pack/filebeat/Jenkinsfile.yml | 7 +++++++ x-pack/functionbeat/Jenkinsfile.yml | 7 +++++++ x-pack/libbeat/Jenkinsfile.yml | 7 +++++++ x-pack/metricbeat/Jenkinsfile.yml | 7 +++++++ 12 files changed, 84 insertions(+) diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 4e4c9e9235d3..8ad6fa7e5d6e 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 681691810d65..a0d58728f66a 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test heartbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" macos: diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index e162af761ef6..8353b4cc22fa 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -17,5 +17,12 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test journalbeat for arm" + labels: + - "arm" + parameters: + - "armTest" unitTest: mage: "mage build unitTest" \ No newline at end of file diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index d4e2e668640d..1506b698963c 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -16,6 +16,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" crosscompile: diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 668aee9b2053..40e737c2f5e6 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test metricbeat for arm" + labels: + - "arm" + parameters: + - "armTest" unitTest: mage: "mage build unitTest" goIntegTest: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index f5e46200d240..e675abec1319 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test packetbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" macos: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 690a7aff42aa..2d3420b5beb4 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 80a43601ad77..a152acaa7692 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/elastic-agent for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" macos: diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 77b337edbfa6..3c7f8593168e 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 309fbcd195d6..d4420f6324db 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/functionbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" macos: diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 28ec5c807c10..25937389fd44 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -17,5 +17,12 @@ stages: mage: "mage build test" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 84d1ae382e68..491fa66684dd 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -17,6 +17,13 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/metricbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. From e018650971e1efb2bc91a980557e9b4e201bc319 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:43:08 +0100 Subject: [PATCH 111/118] Reorder with alphabetic order --- auditbeat/Jenkinsfile.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 2a98cb03e52d..8c03dd42f620 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,10 +13,6 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - build: - mage: "mage build test" - crosscompile: - make: "make -C auditbeat crosscompile" arm: mage: "mage build test" platforms: ## override default label in this specific stage. @@ -28,6 +24,10 @@ stages: - "arm" parameters: - "armTest" + build: + mage: "mage build test" + crosscompile: + make: "make -C auditbeat crosscompile" macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From de33bc22e7bb75bbd9b56f076b2f5835b905c1be Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 15:45:31 +0100 Subject: [PATCH 112/118] Use unitTest since ITs require docker --- auditbeat/Jenkinsfile.yml | 2 +- filebeat/Jenkinsfile.yml | 2 +- heartbeat/Jenkinsfile.yml | 2 +- libbeat/Jenkinsfile.yml | 2 +- packetbeat/Jenkinsfile.yml | 2 +- x-pack/elastic-agent/Jenkinsfile.yml | 2 +- x-pack/functionbeat/Jenkinsfile.yml | 2 +- x-pack/libbeat/Jenkinsfile.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 8c03dd42f620..8952900f6767 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 8ad6fa7e5d6e..9f4cb2fae058 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index a0d58728f66a..2a706080ab77 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 1506b698963c..1c708f4608e7 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index e675abec1319..864d8f5a9c94 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index a152acaa7692..e1aa20528258 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index d4420f6324db..20b77e971303 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 25937389fd44..513038fc2c19 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "linux && ubuntu-18" ## default label for all the stages stages: arm: - mage: "mage build test" + mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" when: ## Aggregate when with the top-level one. From 28301826915ca8b1122819ac4c41f725e4785e4e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 16:54:54 +0100 Subject: [PATCH 113/118] Disable arm builds in metricbeats Caused by #21301 and #21304 --- metricbeat/Jenkinsfile.yml | 11 ----------- x-pack/metricbeat/Jenkinsfile.yml | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 40e737c2f5e6..1219a27af77f 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,17 +13,6 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - arm: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "arm" - when: ## Aggregate when with the top-level one. - comments: - - "/test metricbeat for arm" - labels: - - "arm" - parameters: - - "armTest" unitTest: mage: "mage build unitTest" goIntegTest: diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 491fa66684dd..2448d43d85b1 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,17 +13,6 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: - arm: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "arm" - when: ## Aggregate when with the top-level one. - comments: - - "/test x-pack/metricbeat for arm" - labels: - - "arm" - parameters: - - "armTest" build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. From 7f2fb7f97708ee28be051e80f32135ff85a10966 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 24 Sep 2020 11:25:15 +0100 Subject: [PATCH 114/118] Cherry-pick 009e958a2d000f707d0ded442ffca2af9d9468da and 8d33c8e295f549ffc83342134908e379616a3430 --- .ci/Jenkinsfile | 648 ------------------------------------------------ Jenkinsfile | 21 +- 2 files changed, 16 insertions(+), 653 deletions(-) delete mode 100644 .ci/Jenkinsfile diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index 80971b24a283..000000000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,648 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@feature/support-arm') _ - -import groovy.transform.Field -/** - This is required to store the stashed id with the test results to be digested with runbld -*/ -@Field def stashedTestReports = [:] - -pipeline { - agent { label 'ubuntu-18 && immutable' } - environment { - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - BASE_DIR = 'src/github.com/elastic/beats' - DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_COMPOSE_VERSION = "1.21.0" - DOCKER_REGISTRY = 'docker.elastic.co' - JOB_GCS_BUCKET = 'beats-ci-temp' - JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - PIPELINE_LOG_LEVEL = 'INFO' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' - TERRAFORM_VERSION = "0.12.24" - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - } - options { - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - quietPeriod(10) - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - } - triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') - } - parameters { - booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') - string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') - booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.') - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc and md changes only - setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) - setEnvVar('GO_VERSION', readFile(".go-version").trim()) - withEnv(["HOME=${env.WORKSPACE}"]) { - retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } - } - } - } - } - stage('Lint'){ - options { skipDefaultCheckout() } - environment { - GOFLAGS = '-mod=readonly' - } - steps { - withGithubNotify(context: 'Lint') { - withBeatsEnv(archive: true) { - dumpVariables() - //cmd(label: 'make check', script: 'make check') - } - } - } - } - stage('Build&Test') { - options { skipDefaultCheckout() } - when { - // Always when running builds on branches/tags - // On a PR basis, skip if changes are only related to docs. - // Always when forcing the input parameter - anyOf { - not { changeRequest() } // If no PR - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - script { - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset']).each { k,v -> - mapParallelTasks["${k}"] = v - } - } - parallel(mapParallelTasks) - } - } - } - post { - always { - dir("${BASE_DIR}"){ - // Archive the markdown files that contain the build reasons - archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md') - } - } - } - } - } - post { - always { - runbld() - } - cleanup { - notifyBuildResult(prComment: true) - } - } -} - -/** -* This method is the one used for running the parallel stages, therefore -* its arguments are passed by the beatsStages step. -*/ -def generateStages(Map args = [:]) { - def projectName = args.project - def changeset = args.changeset - def mapParallelStages = [:] - def fileName = "${projectName}/Jenkinsfile.yml" - if (fileExists(fileName)) { - def content = readYaml(file: fileName) - // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. - if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) - } - } else { - log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") - } - return mapParallelStages -} - -def cloud(Map args = [:]) { - node(args.label) { - startCloudTestEnv(name: args.directory, dirs: args.dirs) - } - withCloudTestEnv() { - try { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) - } finally { - terraformCleanup(name: args.directory, dir: args.directory) - } - } -} - -def k8sTest(Map args = [:]) { - def versions = args.versions - node(args.label) { - versions.each{ v -> - stage("${args.context} ${v}"){ - withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "${args.context} ${v}") { - withBeatsEnv(archive: false, withModule: false) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } - try { - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - } finally { - sh(label: 'Delete cluster', script: 'kind delete cluster') - } - } - } - } - } - } - } -} - -/** -* This method runs the given command supporting two kind of scenarios: -* - make -C then the dir(location) is not required, aka by disaling isMage: false -* - mage then the dir(location) is required, aka by enabling isMage: true. -*/ -def target(Map args = [:]) { - def context = args.context - def command = args.command - def directory = args.get('directory', '') - def withModule = args.get('withModule', false) - def isMage = args.get('isMage', false) - node(args.label) { - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { - dumpVariables() - // make commands use -C while mage commands require the dir(folder) - // let's support this scenario with the location variable. - dir(isMage ? directory : '') { - cmd(label: "${command}", script: "${command}") - } - } - } - } -} - -/** -* This method wraps all the environment setup and pre-requirements to run any commands. -*/ -def withBeatsEnv(Map args = [:], Closure body) { - def archive = args.get('archive', true) - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - - def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags - - if(isUnix()) { - if (isArm() && is64arm()) { - // TODO: nodeOS() should support ARM - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64" - gox_flags = '-arch arm' - } else { - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" - gox_flags = '-arch amd64' - } - path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" - magefile = "${WORKSPACE}/.magefile" - pythonEnv = "${WORKSPACE}/python-env" - testResults = '**/build/TEST*.xml' - artifacts = '**/build/TEST*.out' - } else { - def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" - path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}" - magefile = "${env.WORKSPACE}\\.magefile" - testResults = "**\\build\\TEST*.xml" - artifacts = "**\\build\\TEST*.out" - gox_flags = '-arch amd64' - } - - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' - withEnv([ - "DOCKER_PULL=0", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "HOME=${env.WORKSPACE}", - "MAGEFILE_CACHE=${magefile}", - "MODULE=${module}", - "PATH=${path}", - "PYTHON_ENV=${pythonEnv}", - "RACE_DETECTOR=true", - "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "GOX_FLAGS=${gox_flags}" - ]) { - if(isDockerInstalled()) { - dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - dir("${env.BASE_DIR}") { - installTools() - if(isUnix()) { - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" - fi''') - } - try { - body() - } finally { - if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) - } - // Tear down the setup for the permamnent workers. - catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { - fixPermissions("${WORKSPACE}") - deleteDir() - } - } - } - } -} - -/** -* This method fixes the filesystem permissions after the build has happenend. The reason is to -* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental -* issues. -*/ -def fixPermissions(location) { - if(isUnix()) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - set +x - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) - } -} - -/** -* This method installs the required dependencies that are for some reason not available in the -* CI Workers. -*/ -def installTools() { - if(isUnix()) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } - } else { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } - } -} - -/** -* This method gathers the module name, if required, in order to run the ITs only if -* the changeset affects a specific module. -* -* For such, it's required to look for changes under the module folder and exclude anything else -* such as asciidoc and png files. -*/ -def getCommonModuleInTheChangeSet(String directory) { - // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') - def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) - def module = '' - - // Transform folder structure in regex format since path separator is required to be escaped - def transformedDirectory = directory.replaceAll('/', '\\/') - def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" - def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" - dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern, exclude: exclude) - } - return module -} - -/** -* This method archives and report the tests output, for such, it searches in certain folders -* to bypass some issues when working with big repositories. -*/ -def archiveTestOutput(Map args = [:]) { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - if (isUnix()) { - fixPermissions("${WORKSPACE}") - } - cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') - dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id) - archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) - } - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") - if (folder.trim()) { - // TODO: nodeOS() should support ARM - def os_suffix = isArm() ? 'linux' : nodeOS() - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix - tar(file: "${name}.tgz", archive: true, dir: folder) - } - } - } -} - -/** -* This method wraps the junit built-in step to archive the test reports that gonna be populated later on -* with the runbld post build step. -*/ -def junitAndStore(Map args = [:]) { - junit(args) - // args.id could be null in some cases, so let's use the currentmilliseconds - def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" - stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) - stashedTestReports[stageName] = stageName -} - -/** -* This method populates the test output using the runbld approach. For such it requires the -* global variable stashedTestReports. -* TODO: should be moved to the shared library -*/ -def runbld() { - catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { - if (stashedTestReports) { - def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${env.BASE_DIR}") { - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } - } - } - sh(label: 'Process JUnit reports with runbld', - script: """\ - ## for debugging purposes - find . -name "TEST-*.xml" - cat >./runbld-script < the worker labels -* - project -> the name of the project that should match with the folder name. -* - content -> the specific stage data in the /Jenkinsfile.yml -* - context -> the name of the stage, normally -(-)? -*/ -class RunCommand extends co.elastic.beats.BeatsFunction { - public RunCommand(Map args = [:]){ - super(args) - } - public run(Map args = [:]){ - def withModule = args.content.get('withModule', false) - if(args?.content?.containsKey('make')) { - steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) - } - if(args?.content?.containsKey('mage')) { - steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) - } - if(args?.content?.containsKey('k8sTest')) { - steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) - } - if(args?.content?.containsKey('cloud')) { - steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) - } - } -} - -/** -* This class retrieves the dependencies of a Go module for such it transforms them in a -* regex pattern. -*/ -class GetProjectDependencies extends co.elastic.beats.BeatsFunction { - public GetProjectDependencies(Map args = [:]){ - super(args) - } - public run(Map args = [:]){ - def output = "" - steps.withEnv(["HOME=${steps.env.WORKSPACE}"]) { - output = steps.sh(label: 'Get vendor dependency patterns', returnStdout: true, - script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") - } - return output?.split('\n').collect{ item -> item as String } - } -} diff --git a/Jenkinsfile b/Jenkinsfile index e7f91ef95e5e..187529a32075 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,6 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' @@ -43,6 +42,7 @@ pipeline { booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') } stages { @@ -220,10 +220,17 @@ def withBeatsEnv(Map args = [:], Closure body) { def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def goRoot, path, magefile, pythonEnv, testResults, artifacts + def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags if(isUnix()) { - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + if (isArm() && is64arm()) { + // TODO: nodeOS() should support ARM + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64" + gox_flags = '-arch arm' + } else { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + gox_flags = '-arch amd64' + } path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" @@ -237,6 +244,7 @@ def withBeatsEnv(Map args = [:], Closure body) { magefile = "${env.WORKSPACE}\\.magefile" testResults = "**\\build\\TEST*.xml" artifacts = "**\\build\\TEST*.out" + gox_flags = '-arch amd64' } deleteDir() @@ -254,7 +262,8 @@ def withBeatsEnv(Map args = [:], Closure body) { "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle" + "TEST_TAGS=${env.TEST_TAGS},oracle", + "GOX_FLAGS=${gox_flags}" ]) { if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") @@ -353,7 +362,9 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() + // TODO: nodeOS() should support ARM + def os_suffix = isArm() ? 'linux' : nodeOS() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix tar(file: "${name}.tgz", archive: true, dir: folder) } } From 1083b98a16c9a94b194159290f5c74b1d4b9786d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 2 Oct 2020 14:46:15 +0100 Subject: [PATCH 115/118] Timout might happen a bit often if we have only one resource for arm --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 60a577632766..5e2a28457890 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') From 2c872339a308ff9255b5537023cdb05290a21c04 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 2 Oct 2020 14:46:25 +0100 Subject: [PATCH 116/118] Enable on branches/tags --- auditbeat/Jenkinsfile.yml | 4 +++- filebeat/Jenkinsfile.yml | 4 +++- heartbeat/Jenkinsfile.yml | 4 +++- journalbeat/Jenkinsfile.yml | 4 +++- libbeat/Jenkinsfile.yml | 2 +- packetbeat/Jenkinsfile.yml | 4 +++- x-pack/auditbeat/Jenkinsfile.yml | 6 ++++-- x-pack/elastic-agent/Jenkinsfile.yml | 4 +++- x-pack/filebeat/Jenkinsfile.yml | 4 +++- x-pack/functionbeat/Jenkinsfile.yml | 2 +- x-pack/libbeat/Jenkinsfile.yml | 4 +++- 11 files changed, 30 insertions(+), 12 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index b2bee97879fb..cd08240c9a3a 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test auditbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" crosscompile: diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index d5c6edebd718..28e4e4a7bc1d 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test filebeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 84ccb3bd97a3..3830e053b1f0 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test heartbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 1d593feb468c..5715712dd4aa 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -17,12 +17,14 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test journalbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags unitTest: mage: "mage build unitTest" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 1c708f4608e7..692400e7253b 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -16,7 +16,7 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test libbeat for arm" labels: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 6e2fee92da9c..1d252b36b314 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test packetbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 959995ec006e..85535e8c9207 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - - "/test auditbeat for arm" + - "/test x-pack/auditbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index c59abcaa0972..d2b3c117f672 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test x-pack/elastic-agent for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 5689b6c57fe5..8282f9a02b07 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -17,13 +17,15 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test x-pack/filebeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 32adf6cbf2a4..8e380b3957b9 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -17,7 +17,7 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## Override the top-level when. comments: - "/test x-pack/functionbeat for arm" labels: diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 513038fc2c19..d510b8432a72 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -17,12 +17,14 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## Aggregate when with the top-level one. + when: ## AOverride the top-level when. comments: - "/test x-pack/libbeat for arm" labels: - "arm" parameters: - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" From fdb0d707fb68a1343b488047027c9d0a6d73f6dd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 2 Oct 2020 14:47:27 +0100 Subject: [PATCH 117/118] Fix typo --- x-pack/libbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index d510b8432a72..ed22a8dfe70d 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -17,7 +17,7 @@ stages: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - "arm" - when: ## AOverride the top-level when. + when: ## Override the top-level when. comments: - "/test x-pack/libbeat for arm" labels: From b2fbc1b280ed16ca3132d28135ed8f3b6cdc398e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Oct 2020 16:07:57 +0100 Subject: [PATCH 118/118] Update Jenkinsfile Co-authored-by: cachedout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2a994d6e9fae..a16292b38ee7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.') + booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`') }