resolve conflicts

.buildkite/hooks/post-checkout

@@ -1,5 +1,18 @@
 #!/bin/bash
 
+# ******************************* WARNING ******************************************
+# This post-checkout hook is not the same as in the rest of repositories e.g. beats
+# because some steps in this pipeline (in PR context) take a very long time and we
+# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every
+# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime
+# and therefore, some steps (e.g. sonarqube) may end up testing a different commit.
+#
+# Running builds from branches or tags (out of PR context) maintains the same behavior
+# as in the rest of the repositories.
+#
+# Reference: https://github.com/elastic/integrations/pull/10397
+# **********************************************************************************
+
 set -euo pipefail
 
 checkout_merge() {
@@ -13,8 +26,15 @@ checkout_merge() {
     fi
 
     git fetch -v origin "${target_branch}"
-    git checkout FETCH_HEAD
-    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then
+        git checkout FETCH_HEAD
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    else
+        # Use the same commit from target branch as in the other steps.
+        echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    fi
 
     # create temporal branch to merge the PR with the target branch
     git checkout -b ${merge_branch}
@@ -44,6 +64,9 @@ PR_COMMIT="${BUILDKITE_COMMIT}"
 PR_ID=${BUILDKITE_PULL_REQUEST}
 MERGE_BRANCH="pr_merge_${PR_ID}"
 
+# This meta-data field is populated in the pre-command hook
+REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "")
+
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"

.buildkite/hooks/pre-command

@@ -29,7 +29,7 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
+AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
 PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
 
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
@@ -74,6 +74,17 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
+    # Get the commit from target branch in the first step (reference-target-branch).
+    # This step MUST be the first one and not run in parallel with any other step to ensure
+    # that there is just one value for this variable
+    if is_pr ; then
+        git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+        commit_main=$(git rev-parse --verify FETCH_HEAD)
+        buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
+    fi
+fi
+
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
@@ -115,19 +126,13 @@ fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
-        ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_SECRET_KEY
-        ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
+        # add the AWS credentials as in the integrations pipeline.
 
         PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
         export PRIVATE_CI_GCS_CREDENTIALS_SECRET
         export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 
-        # Environment variables required by the service deployer
-        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
-        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
 
@@ -142,11 +147,18 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_USERNAME_SECRET
-    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_EMAIL_SECRET
     GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_TOKEN
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" || "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "report-failed-tests" ]]; then
+    export GITHUB_TOKEN="${VAULT_GITHUB_TOKEN}"
+  fi
+fi
+

.buildkite/pipeline.schedule-daily.yml

@@ -2,7 +2,7 @@
 name: integrations-schedule-daily
 
 env:
-  SETUP_GVM_VERSION: "v0.5.1"
+  SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
 
 # The pipeline is triggered by the scheduler every day
@@ -20,22 +20,19 @@ steps:
     build:
       env:
         SERVERLESS: "false"
-        SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
-        # STACK_VERSION: 7.17-SNAPSHOT # Using 7.17.19 till https://github.com/elastic/fleet-server/issues/3435 is solved.
-        STACK_VERSION: 7.17.19
+        STACK_VERSION: 7.17.23-SNAPSHOT
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.14"
+  - label: "Check integrations local stacks - Stack Version v8.15"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
-        SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.14.0-SNAPSHOT
+        STACK_VERSION: 8.15.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"

.buildkite/pipeline.serverless.yml

@@ -4,11 +4,14 @@ env:
   SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
-  DOCKER_VERSION: "false"
+  DOCKER_VERSION: "false" # not required to set since system tests are not running yet
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.29.0'
+  K8S_VERSION: 'v1.30.0'
   YQ_VERSION: 'v4.35.2'
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  GH_CLI_VERSION: "2.29.0"
+  # This pipeline is intended to test packages with Elastic Serverless
+  SERVERLESS: true
 
   # Elastic package settings
   # Manage docker output/logs
@@ -19,6 +22,10 @@ env:
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
   # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
   ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+  # Enable independent Elastic Agents for all packages
+  ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
+  # Set maximum number of parallel tests to run if package allows it
+  ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
 
 steps:
   - input: "Input values for the variables"
@@ -51,7 +58,6 @@ steps:
     command: ".buildkite/scripts/test_integrations_with_serverless.sh"
     timeout_in_minutes: 240
     env:
-      SERVERLESS: true
       FORCE_CHECK_ALL: true
       UPLOAD_SAFE_LOGS: 1
     agents:
@@ -74,3 +80,16 @@ steps:
           artifacts: "build/test-results/*.xml"
     agents:
       provider: "gcp"  # junit plugin requires docker
+
+  - label: ":github: Report failed tests"
+    key: report-failed-tests
+    command: ".buildkite/scripts/report_issues.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    # not fail build if this step fails
+    soft_fail: true
+    # run this step when if it is triggered by the daily job
+    if: >
+      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/pipeline.yml

@@ -5,7 +5,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.29.0'
+  K8S_VERSION: 'v1.30.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
@@ -20,15 +20,28 @@ env:
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
   # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
   ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+  # Enable independent Elastic Agents for all packages
+  ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
+  # Set maximum number of parallel tests to run if package allows it
+  ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
 
 steps:
+  - label: "Get reference from target branch"
+    key: "reference-target-branch"
+    command: "echo 'Get reference from main'"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+
   - label: ":white_check_mark: Check go sources"
     key: "check"
     command: ".buildkite/scripts/check_sources.sh"
     agents:
       image: "${LINUX_AGENT_IMAGE}"
       cpu: "8"
       memory: "4G"
+    depends_on:
+      - step: "reference-target-branch"
+        allow_failure: false
 
   - label: "Trigger integrations"
     key: "test-integrations"
@@ -53,10 +66,14 @@ steps:
       memory: "4G"
 
   - label: ":sonarqube: Continuous Code Inspection"
+    soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
+    timeout_in_minutes: 120
     env:
       VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+      cpu: "8"
+      memory: "4G"
     command: ".buildkite/scripts/run_sonar_scanner.sh"
     artifact_paths:
       - build/test-coverage/coverage_merged.xml
@@ -68,3 +85,16 @@ steps:
           failed-download-exit-code: 0 # Not fail the build in case there are no XML files
     agents:
       provider: "gcp"  # junit plugin requires docker
+
+  - label: ":github: Report failed tests"
+    key: report-failed-tests
+    command: ".buildkite/scripts/report_issues.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    # not fail build if this step fails
+    soft_fail: true
+    # run this step when if it is triggered by the daily job
+    if: >
+      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/scripts/build_packages.sh

@@ -82,7 +82,7 @@ build_packages() {
 }
 
 if [ "${SKIP_PUBLISHING}" == "true" ] ; then
-    echo "packageStoragePublish: skipping because skip_publishing param is ${SKIP_PUBLISHING}"
+    echo "packageStoragePublish: skipping because SKIP_PUBLISHING environment variable is ${SKIP_PUBLISHING}"
     exit 0
 fi
 

.buildkite/scripts/check_sources.sh

@@ -7,6 +7,6 @@ set -euo pipefail
 add_bin_path
 with_mage
 
-mage -debug check
+mage -v check
 
 check_git_diff

.buildkite/scripts/common.sh

@@ -692,7 +692,7 @@ is_pr_affected() {
 
     echo "[${package}] git-diff: check non-package files"
     commit_merge=$(git merge-base "${from}" "${to}")
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|docs/)' ; then
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|README.md|docs/)' ; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi
@@ -819,6 +819,9 @@ test_package_in_serverless() {
     if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
         return 1
     fi
+    if ! ${ELASTIC_PACKAGE_BIN} test policy ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
+        return 1
+    fi
     echo ""
     return 0
 }

.buildkite/scripts/report_issues.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+junit_folder="test-results"
+test_results_folder="build/${junit_folder}"
+buildkite_pattern="${test_results_folder}/*.xml"
+
+download_test_results() {
+    mkdir -p "${test_results_folder}"
+
+    if ! buildkite-agent artifact download "${buildkite_pattern}" . ; then
+        message="Could not download XML artifacts. Skip creating issues."
+        echo "--- :boom: ${message}"
+        buildkite-agent annotate \
+            "[Report Failed Tests] ${message}" \
+            --context "ctx-report-failed-tests-no-files" \
+            --style "warning"
+        return 1
+    fi
+    return 0
+}
+
+if running_on_buildkite ; then
+    echo "--- Installing tools"
+    add_bin_path
+    with_mage
+    with_github_cli # to list, create and update issues
+
+    echo "--- Download Test Results"
+    if ! download_test_results ; then
+        exit 0
+    fi
+fi
+
+echo "--- Create GitHub Issues for failed tests"
+mage -v ReportFailedTests build/test-results
+