Merge branch 'main' into srvls/disable-bfetch

.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml

@@ -20,6 +20,7 @@ spec:
     spec:
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
       branch_configuration: main 8.14 7.17
       default_branch: main

.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml

@@ -21,6 +21,7 @@ spec:
       env:
         RELEASE_BUILD: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
       branch_configuration: 7.17 8.14
       repository: elastic/kibana

.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml

@@ -21,6 +21,7 @@ spec:
       env:
         RELEASE_BUILD: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
       branch_configuration: '8.14'
       default_branch: main

.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml → .buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -2,10 +2,10 @@
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: bk-kibana-serverless-secsol-qg-api-integration
-  description: Runs the serverless security solution api integration tests for the Quality Gate
+  name: bk-kibana-pull-request
+  description: Runs manually for pull requests
   links:
-    - url: 'https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-api-integration'
+    - url: 'https://buildkite.com/elastic/kibana-pull-request'
       title: Pipeline link
 spec:
   type: buildkite-pipeline
@@ -15,19 +15,24 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: kibana / serverless / security-solution-quality-gate / api-integration
-      description: Runs the serverless security solution api integration tests for the Quality Gate
+      name: kibana / pull request
+      description: Runs manually for pull requests
     spec:
-      env: {}
+      env:
+        PR_COMMENTS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
+        GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
       allow_rebuilds: true
       branch_configuration: ''
+      cancel_intermediate_builds: true
       default_branch: main
       repository: elastic/kibana
-      pipeline_file: .buildkite/pipelines/security_solution/api_integration.yml
+      pipeline_file: .buildkite/scripts/pipelines/pull_request/pipeline.sh
       skip_intermediate_builds: false
       provider_settings:
         build_branches: false
-        build_pull_requests: false
+        build_pull_requests: true
         publish_commit_status: false
         trigger_mode: none
         build_tags: false
@@ -36,8 +41,6 @@ spec:
       teams:
         everyone:
           access_level: BUILD_AND_READ
-        security-engineering-productivity:
-          access_level: MANAGE_BUILD_AND_READ
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ
         appex-qa:

.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml

@@ -20,6 +20,7 @@ spec:
     spec:
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: false
       branch_configuration: main
       default_branch: main

.buildkite/pipeline-resource-definitions/locations.yml

@@ -24,9 +24,9 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Defend Workflows"
-      description: "[MKI] Executes Cypress tests for the Defend Workflows team"
+      description: "[MKI] Executes Cypress and API tests for the Defend Workflows team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Detection Engine"
-      description: "[MKI] Executes Cypress tests for the Detection Engine team"
+      description: "[MKI] Executes Cypress and API tests for the Detection Engine team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Entity Analytics"
-      description: "[MKI] Executes Cypress tests for the Entity Analytics team"
+      description: "[MKI] Executes Cypress and API tests for the Entity Analytics team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Explore"
-      description: "[MKI] Executes Cypress tests for the Explore team"
+      description: "[MKI] Executes Cypress and API tests for the Explore team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Gen Ai"
-      description: "[MKI] Executes Cypress tests for the Gen AI team"
+      description: "[MKI] Executes Cypress and API tests for the Gen AI team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Investigations"
-      description: "[MKI] Executes Cypress tests for the Investigations team"
+      description: "[MKI] Executes Cypress and API tests for the Investigations team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml

.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml

@@ -13,7 +13,7 @@ spec:
     kind: Pipeline
     metadata:
       name: "Kibana / Serverless / Security Solution Quality Gate / Rule Management"
-      description: "[MKI] Executes Cypress tests for the Rule Management team"
+      description: "[MKI] Executes Cypress and API tests for the Rule Management team"
     spec:
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -27,7 +27,7 @@ const getAgentRule = (queueName: string = 'n2-4-spot') => {
     return {
       provider: 'gcp',
       image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-qa',
+      imageProject: 'elastic-images-prod',
       machineType: `${kind}-standard-${cores}`,
       preemptible: spot === 'spot',
     };

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -48,6 +48,7 @@ steps:
           provider: gcp
           machineType: n2-standard-16
         key: build
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
         depends_on: pre-build
         timeout_in_minutes: 60
         retry:

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on:
       - build
       - quick_checks

.buildkite/pipelines/pull_request/base.yml

@@ -3,14 +3,21 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
@@ -22,7 +29,11 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
     key: quick_checks
     timeout_in_minutes: 60
     retry:
@@ -35,7 +46,10 @@ steps:
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -48,7 +62,10 @@ steps:
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@@ -62,7 +79,11 @@ steps:
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      queue: n2-8-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
     key: linting
     timeout_in_minutes: 60
     retry:
@@ -73,7 +94,11 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     key: check_types
     timeout_in_minutes: 70
     retry:
@@ -84,7 +109,11 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: linting_with_types
     timeout_in_minutes: 90
     retry:
@@ -96,7 +125,11 @@ steps:
     label: 'Checks'
     key: checks
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -106,7 +139,11 @@ steps:
   - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     key: build_api_docs
     timeout_in_minutes: 90
     retry:

.buildkite/pipelines/pull_request/build_project.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
     label: 'Build Project Image'
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     timeout_in_minutes: 60
     soft_fail: true
     retry:

.buildkite/pipelines/pull_request/check_next_docs.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/next_docs/build_and_validate_docs.sh
     label: 'Build and Validate Next Docs'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 30
     retry:
       automatic:

.buildkite/pipelines/pull_request/deploy_cloud.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
     label: 'Build and Deploy to Cloud'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
     depends_on:
       - build
       - quick_checks

.buildkite/pipelines/pull_request/deploy_project.yml

@@ -3,7 +3,11 @@ steps:
     label: 'Build Project Image'
     key: build_project_image
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     timeout_in_minutes: 60
     soft_fail: true
     retry:
@@ -13,7 +17,11 @@ steps:
   - command: .buildkite/scripts/steps/serverless/deploy.sh
     label: 'Deploy Project'
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 10
     depends_on:
       - build_project_image

.buildkite/pipelines/pull_request/exploratory_view_plugin.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/exploratory_view_plugin.sh
     label: 'Exploratory View @elastic/synthetics Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on:
       - build
       - quick_checks