resolved merge conflicts

.buildkite/pipeline.schedule-daily.yml

@@ -26,7 +26,7 @@ steps:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.15"
+  - label: "Check integrations local stacks - Stack Version v8.16"
     trigger: "integrations"
     build:
       env:

.buildkite/pipeline.serverless.yml

@@ -76,8 +76,10 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
+          failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 

.buildkite/pipeline.yml

@@ -80,9 +80,10 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
           failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
       "pipelineSlug": "integrations",
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
-      "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]"],
+      "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]", "github-actions[bot]"],
       "set_commit_status": true,
       "build_on_commit": true,
       "build_on_comment": true,

.buildkite/scripts/backport_branch.sh

@@ -17,7 +17,7 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default ${DRY_RUN:-"true"})"
 BASE_COMMIT="$(buildkite-agent meta-data get BASE_COMMIT --default ${BASE_COMMIT:-""})"
 PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default ${PACKAGE_NAME:-""})"
 PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default ${PACKAGE_VERSION:-""})"
-REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get PREMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
+REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get REMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
 
 if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then
   buildkite-agent annotate "The variables **PACKAGE_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning"

.buildkite/scripts/find_oldest_supported_version.py

@@ -2,14 +2,27 @@
 import argparse
 import requests
 import sys
-import yaml
 import unittest
+import yaml
 
-VERSION_URL = "https://artifacts-api.elastic.co/v1/versions?x-elastic-no-kpi=true"
+from requests.adapters import HTTPAdapter, Retry
+
+ARTIFACTS_URL = "https://artifacts-api.elastic.co"
+VERSION_URL = ARTIFACTS_URL + "/v1/versions?x-elastic-no-kpi=true"
 
 
 def fetch_version():
-    return requests.get(VERSION_URL).json()
+    # Retry forever on connection or 500 errors, assume the artifacts API
+    # will come back. If it doesn't come back we cannot continue executing
+    # jobs in any case.
+    retries = Retry(
+        total=None,
+        backoff_factor=0.5,
+        status_forcelist=[500, 502, 503, 504],
+    )
+    session = requests.Session()
+    session.mount(ARTIFACTS_URL, HTTPAdapter(max_retries=retries))
+    return session.get(VERSION_URL).json()
 
 
 def find_oldest_supported_version(kibana_version_condition: str) -> str:

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -50,6 +50,7 @@ for package in ${PACKAGE_LIST}; do
     - label: "Check integrations ${package}"
       key: "test-integrations-${package}"
       command: ".buildkite/scripts/test_one_package.sh ${package} ${from} ${to}"
+      timeout_in_minutes: 240
       agents:
         provider: gcp
         image: ${IMAGE_UBUNTU_X86_64}

.github/CODEOWNERS

@@ -8,6 +8,7 @@
 # Package owners below.
 # Please keep the list sorted.
 /packages/1password @elastic/security-service-integrations
+/packages/abnormal_security @elastic/security-service-integrations
 /packages/activemq @elastic/obs-infraobs-integrations
 /packages/airflow @elastic/obs-infraobs-integrations
 /packages/akamai @elastic/security-service-integrations
@@ -158,6 +159,7 @@
 /packages/darktrace @elastic/security-service-integrations
 /packages/ded @elastic/ml-ui @elastic/sec-applied-ml
 /packages/dga @elastic/ml-ui @elastic/sec-applied-ml
+/packages/digital_guardian @elastic/security-service-integrations
 /packages/docker @elastic/obs-cloudnative-monitoring
 /packages/elastic_agent @elastic/elastic-agent
 /packages/elastic_package_registry @elastic/ecosystem
@@ -205,6 +207,7 @@
 /packages/github @elastic/security-service-integrations
 /packages/gitlab @elastic/security-service-integrations
 /packages/golang @elastic/obs-infraobs-integrations
+/packages/goflow2 @elastic/sec-deployment-and-devices
 /packages/google_cloud_storage @elastic/security-service-integrations
 /packages/google_scc @elastic/security-service-integrations
 /packages/google_workspace @elastic/security-service-integrations
@@ -328,7 +331,7 @@
 /packages/sysmon_linux @elastic/sec-linux-platform
 /packages/system @elastic/obs-infraobs-integrations
 /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
-/packages/system/data_stream/auth @elastic/sec-windows-platform
+/packages/system/data_stream/auth @elastic/sec-linux-platform
 /packages/system/data_stream/security @elastic/sec-windows-platform
 /packages/system/data_stream/application @elastic/sec-windows-platform
 /packages/system/data_stream/core @elastic/obs-infraobs-integrations

.github/ISSUE_TEMPLATE/integration_bug.yml

@@ -293,6 +293,12 @@ body:
 
     validations:
       required: true
+  - type: input
+    id: data_stream_dataset_name
+    attributes:
+      label: Dataset Name
+      description: Which [dataset](https://www.elastic.co/guide/en/fleet/current/data-streams.html) in the integration package are you having issues with?
+      placeholder: ex. nginx.access
   - type: input
     id: integration_version
     attributes:

.github/workflows/bump-elastic-stack-version.yml

@@ -7,7 +7,7 @@ on:
     - cron: '0 1 * * 1-5'
   pull_request:
     paths:
-      - .github/updatecli.d/*
+      - .github/workflows/updatecli/**
       - .github/workflows/bump-elastic-stack-version.yml
 
 permissions:
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@v2.62.0
+        uses: updatecli/updatecli-action@a0c478c868a71d3e239a65714de69450aa1ee2c6 #v2.65.0
 
       - name: Select diff action
         if: ${{ github.event_name == 'pull_request' }}
@@ -37,14 +37,8 @@ jobs:
         run: |
           echo "UPDATECLI_ACTION=apply" >> $GITHUB_ENV
 
-      - name: Update latest testing 7.x stack version
+      - name: Update latest testing stack versions
         # --experimental needed for commitusingapi option.
-        run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli.d/bump-latest-7x-version.yml --values .github/workflows/updatecli.d/scm.yml
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Update latest testing stack version
-        # --experimental needed for commitusingapi option.
-        run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli.d/bump-latest-snapshot-version.yml --values .github/workflows/updatecli.d/scm.yml
+        run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli/updatecli.d --values .github/workflows/updatecli/values.d/scm.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/updatecli.d/bump-latest-7x-version.yml → .github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml

@@ -19,6 +19,7 @@ scms:
       owner: '{{ .scm.owner }}'
       repository: '{{ .scm.repository }}'
       user: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
       commitusingapi: true
       branch: main

.github/workflows/updatecli.d/bump-latest-snapshot-version.yml → .github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml

@@ -19,6 +19,7 @@ scms:
       owner: '{{ .scm.owner }}'
       repository: '{{ .scm.repository }}'
       user: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
       commitusingapi: true
       branch: main
@@ -30,6 +31,16 @@ sources:
     spec:
       file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
       key: .version
+  latestSnapshotMajorMinor:
+    name: Get latest snapshort major and minor
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      key: .version
+    transformers:
+      - findsubmatch:
+          pattern: '(^\d+\.\d+)\.[^\s]+$'
+          captureindex: 1
 
 targets:
   update-snapshot:
@@ -41,3 +52,13 @@ targets:
       file: '.buildkite/pipeline.schedule-daily.yml'
       matchpattern: '(STACK_VERSION:) 8\.[^\s]*\.[^\s]*'
       replacepattern: '$1 {{ source "latestSnapshot" }}'
+
+  update-snapshot-label:
+    name: '[updatecli] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
+    kind: file
+    sourceid: latestSnapshotMajorMinor
+    scmid: default
+    spec:
+      file: '.buildkite/pipeline.schedule-daily.yml'
+      matchpattern: '(Stack Version) v8\.\d+"'
+      replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"'