Merge branch 'main' into docs/8.0.0-rc1-rn

.backportrc.json

@@ -3,6 +3,7 @@
   "targetBranchChoices": [
     { "name": "main", "checked": true },
     "8.0",
+    "7.17",
     "7.16",
     "7.15",
     "7.14",

.buildkite/pipelines/bazel_cache.yml

@@ -0,0 +1,33 @@
+steps:
+  - label: ':pipeline: Create pipeline with priority'
+    concurrency_group: bazel_macos
+    concurrency: 1
+    concurrency_method: eager
+    # We have to use a dynamic pipeline to set PRIORITY at runtime based on the branch
+    # We want the main branch to be prioritized higher than other branches
+    # The other option would be to have a slightly different version of this yaml live on different branches
+    # But this makes backports easier / everything cleaner
+    command: |
+      if [[ "$${BUILDKITE_BRANCH}" == "$${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" ]]; then
+        export PRIORITY=1
+      else
+        export PRIORITY=0
+      fi
+      buildkite-agent pipeline upload <<YAML
+      steps:
+      - command: .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
+        label: Bootstrap (MacOS x86)
+        priority: $${PRIORITY}
+        agents:
+          queue: macos-x86
+        timeout_in_minutes: 60
+        concurrency_group: bazel_macos
+        concurrency: 1
+        concurrency_method: eager
+      - command: .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
+        label: Bootstrap (MacOS ARM)
+        priority: $${PRIORITY}
+        agents:
+          queue: macos-arm
+        timeout_in_minutes: 60
+      YAML

.buildkite/pipelines/flaky_tests/groups.json

@@ -0,0 +1,34 @@
+{
+  "groups": [
+    {
+      "key": "oss/cigroup",
+      "name": "OSS CI Group",
+      "ciGroups": 12
+    },
+    {
+      "key": "oss/firefox",
+      "name": "OSS Firefox"
+    },
+    {
+      "key": "oss/accessibility",
+      "name": "OSS Accessibility"
+    },
+    {
+      "key": "xpack/cigroup",
+      "name": "Default CI Group",
+      "ciGroups": 27
+    },
+    {
+      "key": "xpack/cigroup/Docker",
+      "name": "Default CI Group Docker"
+    },
+    {
+      "key": "xpack/firefox",
+      "name": "Default Firefox"
+    },
+    {
+      "key": "xpack/accessibility",
+      "name": "Default Accessibility"
+    }
+  ]
+}

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -1,3 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>} */ (
+  require('./groups.json').groups
+);
+
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -7,38 +19,31 @@ const stepInput = (key, nameOfSuite) => {
   };
 };
 
-const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 27;
-
 const inputs = [
   {
     key: 'ftsr-override-count',
     text: 'Override for all suites',
-    default: 0,
+    default: '0',
     required: true,
   },
 ];
 
-for (let i = 1; i <= OSS_CI_GROUPS; i++) {
-  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+for (const group of groups) {
+  if (!group.ciGroups) {
+    inputs.push(stepInput(group.key, group.name));
+  } else {
+    for (let i = 1; i <= group.ciGroups; i++) {
+      inputs.push(stepInput(`${group.key}/${i}`, `${group.name} ${i}`));
+    }
+  }
 }
 
-inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
-inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
-
-for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
-  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
-}
-
-inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
-inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
-inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
-
 const pipeline = {
   steps: [
     {
       input: 'Number of Runs - Click Me',
       fields: inputs,
+      if: `build.env('KIBANA_FLAKY_TEST_RUNNER_CONFIG') == null`,
     },
     {
       wait: '~',

.buildkite/pipelines/flaky_tests/runner.js

@@ -1,37 +1,95 @@
-const { execSync } = require('child_process');
-
-const keys = execSync('buildkite-agent meta-data keys')
-  .toString()
-  .split('\n')
-  .filter((k) => k.startsWith('ftsr-suite/'));
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
 
-const overrideCount = parseInt(
-  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
-);
+const { execSync } = require('child_process');
 
 const concurrency = 25;
+const defaultCount = concurrency * 2;
 const initialJobs = 3;
 
-let totalJobs = initialJobs;
+function getTestSuitesFromMetadata() {
+  const keys = execSync('buildkite-agent meta-data keys')
+    .toString()
+    .split('\n')
+    .filter((k) => k.startsWith('ftsr-suite/'));
+
+  const overrideCount = execSync(`buildkite-agent meta-data get 'ftsr-override-count'`)
+    .toString()
+    .trim();
+
+  const testSuites = [];
+  for (const key of keys) {
+    if (!key) {
+      continue;
+    }
+
+    const value =
+      overrideCount && overrideCount !== '0'
+        ? overrideCount
+        : execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+    const count = value === '' ? defaultCount : parseInt(value);
+    testSuites.push({
+      key: key.replace('ftsr-suite/', ''),
+      count: count,
+    });
+  }
 
-const testSuites = [];
-for (const key of keys) {
-  if (!key) {
-    continue;
+  return testSuites;
+}
+
+function getTestSuitesFromJson(json) {
+  const fail = (errorMsg) => {
+    console.error('+++ Invalid test config provided');
+    console.error(`${errorMsg}: ${json}`);
+    process.exit(1);
+  };
+
+  let parsed;
+  try {
+    parsed = JSON.parse(json);
+  } catch (error) {
+    fail(`JSON test config did not parse correctly`);
   }
 
-  const value =
-    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+  if (!Array.isArray(parsed)) {
+    fail(`JSON test config must be an array`);
+  }
 
-  const count = value === '' ? defaultCount : parseInt(value);
-  totalJobs += count;
+  /** @type {Array<{ key: string, count: number }>} */
+  const testSuites = [];
+  for (const item of parsed) {
+    if (typeof item !== 'object' || item === null) {
+      fail(`testSuites must be objects`);
+    }
+    const key = item.key;
+    if (typeof key !== 'string') {
+      fail(`testSuite.key must be a string`);
+    }
+    const count = item.count;
+    if (typeof count !== 'number') {
+      fail(`testSuite.count must be a number`);
+    }
+    testSuites.push({
+      key,
+      count,
+    });
+  }
 
-  testSuites.push({
-    key: key.replace('ftsr-suite/', ''),
-    count: count,
-  });
+  return testSuites;
 }
 
+const testSuites = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG
+  ? getTestSuitesFromJson(process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG)
+  : getTestSuitesFromMetadata();
+
+const totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
+
 if (totalJobs > 500) {
   console.error('+++ Too many tests');
   console.error(

.buildkite/scripts/build_kibana.sh

@@ -7,6 +7,8 @@ export KBN_NP_PLUGINS_BUILT=true
 echo "--- Build Kibana Distribution"
 if [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-all-platforms"* ]]; then
   node scripts/build --all-platforms --skip-os-packages
+elif [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-os-packages"* ]]; then
+  node scripts/build --all-platforms
 else
   node scripts/build
 fi
@@ -26,7 +28,7 @@ if [[ "${GITHUB_PR_LABELS:-}" == *"ci:deploy-cloud"* ]]; then
     --docker-tag-qualifier="$GIT_COMMIT" \
     --docker-push \
     --skip-docker-ubi \
-    --skip-docker-centos \
+    --skip-docker-ubuntu \
     --skip-docker-contexts
 
   CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)

.buildkite/scripts/common/env.sh

@@ -76,8 +76,14 @@ export GIT_BRANCH="${BUILDKITE_BRANCH:-}"
 export FLEET_PACKAGE_REGISTRY_PORT=6104
 export TEST_CORS_SERVER_PORT=6105
 
-export DETECT_CHROMEDRIVER_VERSION=true
-export CHROMEDRIVER_FORCE_DOWNLOAD=true
+# Mac agents currently don't have Chrome
+if [[ "$(which google-chrome-stable)" || "$(which google-chrome)" ]]; then
+  echo "Chrome detected, setting DETECT_CHROMEDRIVER_VERSION=true"
+  export DETECT_CHROMEDRIVER_VERSION=true
+  export CHROMEDRIVER_FORCE_DOWNLOAD=true
+else
+  echo "Chrome not detected, installing default chromedriver binary for the package version"
+fi
 
 export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this
 

.buildkite/scripts/common/setup_bazel.sh

@@ -2,9 +2,6 @@
 
 source .buildkite/scripts/common/util.sh
 
-KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
-export KIBANA_BUILDBUDDY_CI_API_KEY
-
 echo "[bazel] writing .bazelrc"
 cat <<EOF > $KIBANA_DIR/.bazelrc
   # Generated by .buildkite/scripts/common/setup_bazel.sh

.buildkite/scripts/common/setup_node.sh

@@ -8,35 +8,67 @@ export NODE_DIR="$CACHE_DIR/node/$NODE_VERSION"
 export NODE_BIN_DIR="$NODE_DIR/bin"
 export YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache"
 
-if [[ ! -d "$NODE_DIR" ]]; then
-  hostArch="$(command uname -m)"
-  case "${hostArch}" in
-    x86_64 | amd64) nodeArch="x64" ;;
-    aarch64) nodeArch="arm64" ;;
-    *) nodeArch="${hostArch}" ;;
-  esac
+## Install node for whatever the current os/arch are
+hostArch="$(command uname -m)"
+case "${hostArch}" in
+  x86_64 | amd64) nodeArch="x64" ;;
+  aarch64) nodeArch="arm64" ;;
+  *) nodeArch="${hostArch}" ;;
+esac
+classifier="$nodeArch.tar.gz"
 
-  nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$nodeArch.tar.gz"
+UNAME=$(uname)
+OS="linux"
+if [[ "$UNAME" = *"MINGW64_NT"* ]]; then
+  OS="win"
+  NODE_BIN_DIR="$HOME/node"
+  classifier="x64.zip"
+elif [[ "$UNAME" == "Darwin" ]]; then
+  OS="darwin"
+fi
+echo " -- Running on OS: $OS"
 
-  echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl"
+nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-${OS}-${classifier}"
 
-  mkdir -p "$NODE_DIR"
-  curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
+echo " -- node: version=v${NODE_VERSION} dir=$NODE_DIR"
+
+echo " -- setting up node.js"
+if [ -x "$NODE_BIN_DIR/node" ] && [ "$("$NODE_BIN_DIR/node" --version)" == "v$NODE_VERSION" ]; then
+  echo " -- reusing node.js install"
 else
-  echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using"
-  ls -alh "$NODE_BIN_DIR"
+  if [ -d "$NODE_DIR" ]; then
+    echo " -- clearing previous node.js install"
+    rm -rf "$NODE_DIR"
+  fi
+
+  echo " -- downloading node.js from $nodeUrl"
+  mkdir -p "$NODE_DIR"
+  if [[ "$OS" == "win" ]]; then
+    nodePkg="$NODE_DIR/${nodeUrl##*/}"
+    curl --silent -L -o "$nodePkg" "$nodeUrl"
+    unzip -qo "$nodePkg" -d "$NODE_DIR"
+    mv "${nodePkg%.*}" "$NODE_BIN_DIR"
+  else
+    curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
+  fi
 fi
 
 export PATH="$NODE_BIN_DIR:$PATH"
 
-
 echo "--- Setup Yarn"
 
 YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")
 export YARN_VERSION
 
 if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
-  npm install -g "yarn@^${YARN_VERSION}"
+  rm -rf "$(npm root -g)/yarn" # in case the directory is in a bad state
+  if [[ ! $(npm install -g "yarn@^${YARN_VERSION}") ]]; then
+    # If this command is terminated early, e.g. because the build was cancelled in buildkite,
+    # a yarn directory is left behind in a bad state that can cause all subsequent installs to fail
+    rm -rf "$(npm root -g)/yarn"
+    echo "Trying again to install yarn..."
+    npm install -g "yarn@^${YARN_VERSION}"
+  fi
 fi
 
 yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"