Merge branch 'main' into canvas_remove_bfetch

.buildkite/pipeline-resource-definitions/kibana-codeql.yml

@@ -0,0 +1,34 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-codeql
+  description: Run CodeQL
+  links:
+    - title: Pipeline link
+      url: https://buildkite.com/elastic/kibana-codeql
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / codeql
+      description: Run CodeQL
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+      repository: elastic/kibana
+      branch_configuration: main
+      default_branch: main
+      pipeline_file: ".buildkite/pipelines/codeql/codeql.yml"
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -44,3 +44,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml

.buildkite/pipeline-utils/buildkite/client.test.ts

@@ -24,9 +24,7 @@ describe('BuildkiteClient', () => {
         id: 'id-1',
         retried_in_job_id: 'id-2',
         state: 'failed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         retried: true,
         exit_status: -1,
         type: 'script',
@@ -35,9 +33,7 @@ describe('BuildkiteClient', () => {
       const retry: Job = {
         id: 'id-2',
         state: 'passed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         type: 'script',
       } as Job;
 
@@ -58,9 +54,7 @@ describe('BuildkiteClient', () => {
         id: 'id-1',
         retried_in_job_id: 'id-2',
         state: 'failed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         retried: true,
         exit_status: 1,
         type: 'script',
@@ -69,9 +63,7 @@ describe('BuildkiteClient', () => {
       const retry: Job = {
         id: 'id-2',
         state: 'passed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         type: 'script',
       } as Job;
 

.buildkite/pipeline-utils/buildkite/client.ts

@@ -285,7 +285,7 @@ export class BuildkiteClient {
         hasRetries = true;
         const isPreemptionFailure =
           job.state === 'failed' &&
-          job.agent?.meta_data?.some((el) => ['spot=true', 'gcp:preemptible=true'].includes(el)) &&
+          job.agent_query_rules?.includes('preemptible=true') &&
           job.exit_status === -1;
 
         if (!isPreemptionFailure) {

.buildkite/pipelines/codeql/codeql.yml

@@ -0,0 +1,2 @@
+steps:
+  - command: echo "Placeholder"

.devcontainer/.env.template

@@ -0,0 +1,4 @@
+# /bin/bash or /bin/zsh (oh-my-zsh is installed by default as well)
+SHELL=/bin/bash
+# Switch to 1 to enable FIPS environment, any other value to disable
+FIPS=0

.devcontainer/Dockerfile

@@ -0,0 +1,69 @@
+FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04
+
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
+ENV HOME=/home/vscode
+ENV NVM_DIR=${HOME}/nvm
+ENV NVM_VERSION=v0.39.1
+ENV KBN_DIR=/workspaces/kibana
+ENV OPENSSL_PATH=${HOME}/openssl
+# Only specific versions are FIPS certified.
+ENV OPENSSL_VERSION='3.0.8'
+
+RUN apt-get update && apt-get install -y curl git zsh locales docker.io perl make gcc xvfb
+
+RUN locale-gen en_US.UTF-8
+
+# Oh My Zsh setup
+RUN if [ ! -d "$HOME/.oh-my-zsh" ]; then \
+  sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"; \
+  fi && \
+  ZSH_CUSTOM=${ZSH_CUSTOM:-~/.oh-my-zsh/custom} && \
+  if [ ! -d "$ZSH_CUSTOM/plugins/zsh-autosuggestions" ]; then \
+  git clone https://github.com/zsh-users/zsh-autosuggestions $ZSH_CUSTOM/plugins/zsh-autosuggestions; \
+  fi && \
+  sed -i 's/plugins=(git)/plugins=(git ssh-agent npm docker zsh-autosuggestions)/' /home/vscode/.zshrc
+
+# Docker-in-Docker setup
+RUN usermod -aG docker vscode
+
+# FIPS setup
+# https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md
+# https://www.openssl.org/docs/man3.0/man7/fips_module.html
+WORKDIR ${HOME}
+
+RUN set -e ; \
+  mkdir -p "${OPENSSL_PATH}"; \
+  curl --retry 8 -S -L -O "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" ; \
+  curl --retry 8 -S -L -O "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz.sha256" ; \
+  echo "$(cat openssl-${OPENSSL_VERSION}.tar.gz.sha256) openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c ; \
+  tar -zxf "openssl-${OPENSSL_VERSION}.tar.gz" ; \
+  rm -rf openssl-${OPENSSL_VERSION}.tar* ; \
+  cd "${OPENSSL_PATH}-${OPENSSL_VERSION}" ; \
+  ./Configure --prefix="${OPENSSL_PATH}" --openssldir="${OPENSSL_PATH}/ssl" --libdir="${OPENSSL_PATH}/lib" shared -Wl,-rpath,${OPENSSL_PATH}/lib enable-fips; \
+  make -j $(nproc) > /dev/null ; \
+  make install > /dev/null ; \
+  rm -rf  "${OPENSSL_PATH}-${OPENSSL_VERSION}" ; \
+  chown -R 1000:1000 "${OPENSSL_PATH}";
+
+WORKDIR ${KBN_DIR}
+
+# Node and NVM setup
+COPY .node-version /tmp/
+RUN mkdir -p $NVM_DIR && \
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh | bash && \
+  . "$NVM_DIR/nvm.sh" && \
+  NODE_VERSION=$(cat /tmp/.node-version) && \
+  nvm install ${NODE_VERSION} && \
+  nvm use ${NODE_VERSION} && \
+  nvm alias default ${NODE_VERSION} && \
+  npm install -g yarn && \
+  echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.bashrc && \
+  echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.zshrc  && \
+  chown -R 1000:1000 "${HOME}/.npm"
+
+# Reload the env everytime a new shell is opened incase the .env file changed.
+RUN echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.bashrc && \
+  echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.zshrc
+
+# This is for documentation. Ports are exposed via devcontainer.json
+EXPOSE 9200 5601 9229 9230 9231

.devcontainer/README.md

@@ -0,0 +1 @@
+See the [dev docs](https://github.com/elastic/kibana/blob/main/dev_docs/getting_started/setting_up_a_development_env.mdx#using-the-kibana-dev-container-optional) for information on using the Kibana Dev Container.

.devcontainer/config/nodejs.cnf

@@ -0,0 +1,28 @@
+##########################################################################
+##                                                                      ##
+## This OpenSSL config is only loaded when running Kibana in FIPS mode. ##
+##                                                                      ##
+## See:                                                                 ##
+## https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md   ##
+## https://www.openssl.org/docs/man3.0/man7/fips_module.html            ##
+##                                                                      ##
+##########################################################################
+
+nodejs_conf = nodejs_init
+.include /home/vscode/openssl/ssl/fipsmodule.cnf
+
+[nodejs_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+default = default_sect
+# The fips section name should match the section name inside the
+# included fipsmodule.cnf.
+fips = fips_sect
+
+[default_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=yes 

.devcontainer/devcontainer.json

@@ -0,0 +1,41 @@
+{
+  "name": "Kibana",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "dbaeumer.vscode-eslint",
+        "ms-azuretools.vscode-docker",
+        "editorconfig.editorconfig",
+        "timonwong.shellcheck",
+        "eamodio.gitlens",
+        "github.vscode-pull-request-github"
+      ]
+    }
+  },
+  "forwardPorts": [
+    9200,
+    5601,
+    9229,
+    9230,
+    9231
+  ],
+  "postStartCommand": "/workspaces/kibana/.devcontainer/scripts/post_start.sh",
+  "remoteUser": "vscode",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "latest",
+      "dockerDashComposeVersion": "latest"
+    },
+    "ghcr.io/devcontainers/features/github-cli:1": {
+      "installDirectlyFromGitHubRelease": true,
+      "version": "latest"
+    },
+    "ghcr.io/kreemer/features/chrometesting:1": {
+      "version": "stable"
+    }
+  }
+}

.devcontainer/scripts/env.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+ENV_PATH="${KBN_DIR}/.devcontainer/.env"
+KBN_CONFIG_FILE="${KBN_DIR}/config/kibana.dev.yml"
+
+setup_fips() {
+  if [ ! -f "$KBN_CONFIG_FILE" ]; then
+    touch "$KBN_CONFIG_FILE"
+  fi
+
+  if [ -n "$FIPS" ] && [ "$FIPS" = "1" ]; then
+    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: true/; t}; $a\xpack.security.experimental.fipsMode.enabled: true' "$KBN_CONFIG_FILE"
+
+    # Patch node_modules so we can start Kibana in dev mode
+    sed -i 's/hashType = hashType || '\''md5'\'';/hashType = hashType || '\''sha1'\'';/g' "${KBN_DIR}/node_modules/file-loader/node_modules/loader-utils/lib/getHashDigest.js"
+    sed -i 's/const hash = createHash("md4");/const hash = createHash("sha1");/g' "${KBN_DIR}/node_modules/webpack/lib/ModuleFilenameHelpers.js"
+    sed -i 's/contentHash: createHash("md4")/contentHash: createHash("sha1")/g' "${KBN_DIR}/node_modules/webpack/lib/SourceMapDevToolPlugin.js"
+
+    export OPENSSL_MODULES="$OPENSSL_PATH/lib/ossl-modules"
+    export NODE_OPTIONS="--enable-fips --openssl-config=$KBN_DIR/.devcontainer/config/nodejs.cnf"
+    echo "FIPS mode enabled"
+    echo "If manually bootstrapping in FIPS mode use: NODE_OPTIONS='' yarn kbn bootstrap"
+  else
+    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: false/; t}; $a\xpack.security.experimental.fipsMode.enabled: false' "$KBN_CONFIG_FILE"
+  fi
+}
+
+setup_shell() {
+  if [ -n "$SHELL" ] && [ -x "$SHELL" ]; then
+    current_shell=$(ps -p $$ -o comm=)
+    desired_shell=$(basename "$SHELL")
+
+    if [ "$current_shell" != "$desired_shell" ]; then
+      sudo chsh -s "$SHELL" vscode
+      exec "$SHELL"
+    fi
+  else
+    echo "Shell is not set or not executable, using bash"
+  fi
+}
+
+if [ -f "$ENV_PATH" ]; then
+  source "$ENV_PATH"
+  setup_fips
+  setup_shell
+else
+  echo ".env file not found, using default values"
+fi

.devcontainer/scripts/post_start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# If FIPS mode is enabled, there can be issues installing some dependencies due to invalid algorithms.
+# So override the NODE_OPTIONS environment variable to disable FIPS mode.
+NODE_OPTIONS='' yarn kbn bootstrap
+
+Xvfb :99 -screen 0 1920x1080x24 &
+export DISPLAY=:99

.eslintrc.js

@@ -928,7 +928,10 @@ module.exports = {
       },
     },
     {
-      files: ['x-pack/plugins/observability_solution/**/*.{ts,tsx}'],
+      files: [
+        'x-pack/plugins/observability_solution/**/*.{ts,tsx}',
+        'x-pack/packages/observability/**/*.{ts,tsx}',
+      ],
       rules: {
         'react-hooks/exhaustive-deps': [
           'error',
@@ -944,6 +947,7 @@ module.exports = {
         'x-pack/plugins/aiops/**/*.tsx',
         'x-pack/plugins/observability_solution/**/*.tsx',
         'src/plugins/ai_assistant_management/**/*.tsx',
+        'x-pack/packages/observability/**/*.{ts,tsx}',
       ],
       rules: {
         '@kbn/telemetry/event_generating_elements_should_be_instrumented': 'error',