Merge branch 'main' into 180444-apm-ui-fix-opentelemetry-agent-names

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -39,8 +39,13 @@ if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-performance-data-set-extraction" ]; t
   node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --skip-warmup
 else
   # pipeline should use bare metal static worker
-  echo "--- Running performance tests"
-  node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+  if [[ -z "${JOURNEYS_GROUP+x}" ]]; then
+    echo "--- Running performance tests"
+    node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+  else
+    echo "--- Running performance tests: '$JOURNEYS_GROUP' group"
+    node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --group "$JOURNEYS_GROUP"
+  fi
 fi
 
 echo "--- Upload journey step screenshots"

.devcontainer/.env.template

@@ -1,4 +1,5 @@
 # /bin/bash or /bin/zsh (oh-my-zsh is installed by default as well)
 SHELL=/bin/bash
-# Switch to 1 to enable FIPS environment, any other value to disable
+# Switch to 1 to enable FIPS environment, any other value to disable,
+# then close and reopen a new terminal to setup the environment
 FIPS=0

.devcontainer/Dockerfile

@@ -49,6 +49,9 @@ WORKDIR ${KBN_DIR}
 
 # Node and NVM setup
 COPY .node-version /tmp/
+
+USER vscode
+
 RUN mkdir -p $NVM_DIR && \
   curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh | bash && \
   . "$NVM_DIR/nvm.sh" && \
@@ -61,6 +64,8 @@ RUN mkdir -p $NVM_DIR && \
   echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.zshrc  && \
   chown -R 1000:1000 "${HOME}/.npm"
 
+USER root
+
 # Reload the env everytime a new shell is opened incase the .env file changed.
 RUN echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.bashrc && \
   echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.zshrc

dev_docs/tutorials/performance/adding_performance_journey.mdx

@@ -89,6 +89,27 @@ simulate real life internet connection. This means that all requests have a fixe
 In order to keep track on performance metrics stability, journeys are run on main branch with a scheduled interval.
 Bare metal machine is used to produce results as stable and reproducible as possible.
 
+#### Running subset of journeys for the PR
+
+Some code changes might affect the Kibana performance and it might be benefitial to run relevant journeys against the PR
+and compare performance metrics vs. the ones on main branch.
+
+In oder to trigger the build for Kibana PR, you can follow these steps:
+
+- Create a new kibana-single-user-performance [build](https://buildkite.com/elastic/kibana-single-user-performance#new)
+- Provide the following arguments:
+  - Branch: `refs/pull/<PR_number>/head`
+  - Under Options, set the environment variable: `JOURNEYS_GROUP=<group_name>`
+
+Currently supported journey groups:
+
+- kibanaStartAndLoad
+- crud
+- dashboard
+- discover
+- maps
+- ml
+
 #### Machine specifications
 
 All benchmarks are run on bare-metal machines with the [following specifications](https://www.hetzner.com/dedicated-rootserver/ex100):

package.json

@@ -1801,7 +1801,7 @@
     "tape": "^5.0.1",
     "terser": "^5.32.0",
     "terser-webpack-plugin": "^4.2.3",
-    "tough-cookie": "^4.1.4",
+    "tough-cookie": "^5.0.0",
     "tree-kill": "^1.2.2",
     "ts-morph": "^15.1.0",
     "tsd": "^0.31.1",

packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx

@@ -14,6 +14,7 @@ import { UiCounterMetricType } from '@kbn/analytics';
 import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import { Draggable } from '@kbn/dom-drag-drop';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { Filter } from '@kbn/es-query';
 import type { SearchMode } from '../../types';
 import { FieldItemButton, type FieldItemButtonProps } from '../../components/field_item_button';
 import {
@@ -200,6 +201,10 @@ export interface UnifiedFieldListItemProps {
    * Item size
    */
   size: FieldItemButtonProps<DataViewField>['size'];
+  /**
+   * Custom filters to apply for the field list, ex: namespace custom filter
+   */
+  additionalFilters?: Filter[];
 }
 
 function UnifiedFieldListItemComponent({
@@ -223,6 +228,7 @@ function UnifiedFieldListItemComponent({
   groupIndex,
   itemIndex,
   size,
+  additionalFilters,
 }: UnifiedFieldListItemProps) {
   const [infoIsOpen, setOpen] = useState(false);
 
@@ -288,6 +294,7 @@ function UnifiedFieldListItemComponent({
           multiFields={multiFields}
           dataView={dataView}
           onAddFilter={addFilterAndClosePopover}
+          additionalFilters={additionalFilters}
         />
 
         {searchMode === 'documents' && multiFields && (

packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item_stats.tsx

@@ -27,10 +27,11 @@ export interface UnifiedFieldListItemStatsProps {
   dataView: DataView;
   multiFields?: Array<{ field: DataViewField; isSelected: boolean }>;
   onAddFilter: FieldStatsProps['onAddFilter'];
+  additionalFilters?: FieldStatsProps['filters'];
 }
 
 export const UnifiedFieldListItemStats: React.FC<UnifiedFieldListItemStatsProps> = React.memo(
-  ({ stateService, services, field, dataView, multiFields, onAddFilter }) => {
+  ({ stateService, services, field, dataView, multiFields, onAddFilter, additionalFilters }) => {
     const querySubscriberResult = useQuerySubscriber({
       data: services.data,
       timeRangeUpdatesType: stateService.creationOptions.timeRangeUpdatesType,
@@ -55,6 +56,11 @@ export const UnifiedFieldListItemStats: React.FC<UnifiedFieldListItemStatsProps>
       [services]
     );
 
+    const filters = useMemo(
+      () => [...(querySubscriberResult.filters ?? []), ...(additionalFilters ?? [])],
+      [querySubscriberResult.filters, additionalFilters]
+    );
+
     if (!hasQuerySubscriberData(querySubscriberResult)) {
       return null;
     }
@@ -63,7 +69,7 @@ export const UnifiedFieldListItemStats: React.FC<UnifiedFieldListItemStatsProps>
       <FieldStats
         services={statsServices}
         query={querySubscriberResult.query}
-        filters={querySubscriberResult.filters}
+        filters={filters}
         fromDate={querySubscriberResult.fromDate}
         toDate={querySubscriberResult.toDate}
         dataViewOrDataViewId={dataView}

packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx

@@ -50,6 +50,7 @@ export type UnifiedFieldListSidebarCustomizableProps = Pick<
   | 'onAddFilter'
   | 'onAddFieldToWorkspace'
   | 'onRemoveFieldFromWorkspace'
+  | 'additionalFilters'
 > & {
   /**
    * All fields: fields from data view and unmapped fields or columns from text-based search
@@ -168,6 +169,7 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
   onDeleteField,
   onToggleSidebar,
   additionalFieldGroups,
+  additionalFilters,
 }) => {
   const { dataViews, core } = services;
   const useNewFieldsApi = useMemo(
@@ -285,6 +287,7 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
             groupName === FieldsGroupNames.SelectedFields ||
             Boolean(selectedFieldsState.selectedFieldsMap[field.name])
           }
+          additionalFilters={additionalFilters}
         />
       </li>
     ),
@@ -304,6 +307,7 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
       onDeleteField,
       workspaceSelectedFieldNames,
       selectedFieldsState.selectedFieldsMap,
+      additionalFilters,
     ]
   );
 

packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar_container.tsx

@@ -117,6 +117,7 @@ const UnifiedFieldListSidebarContainer = memo(
         prependInFlyout,
         variant = 'responsive',
         onFieldEdited,
+        additionalFilters,
       } = props;
       const [stateService] = useState<UnifiedFieldListSidebarContainerStateService>(
         createStateService({ options: getCreationOptions() })
@@ -151,11 +152,16 @@ const UnifiedFieldListSidebarContainer = memo(
       const searchMode: SearchMode | undefined = querySubscriberResult.searchMode;
       const isAffectedByGlobalFilter = Boolean(querySubscriberResult.filters?.length);
 
+      const filters = useMemo(
+        () => [...(querySubscriberResult.filters ?? []), ...(additionalFilters ?? [])],
+        [querySubscriberResult.filters, additionalFilters]
+      );
+
       const { isProcessing, refetchFieldsExistenceInfo } = useExistingFieldsFetcher({
         disableAutoFetching: stateService.creationOptions.disableFieldsExistenceAutoFetching,
         dataViews: searchMode === 'documents' && dataView ? [dataView] : [],
         query: querySubscriberResult.query,
-        filters: querySubscriberResult.filters,
+        filters,
         fromDate: querySubscriberResult.fromDate,
         toDate: querySubscriberResult.toDate,
         services,

renovate.json

@@ -493,6 +493,15 @@
       "labels": ["Team: Sec Eng Productivity", "release_note:skip", "backport:all-open"],
       "minimumReleaseAge": "7 days",
       "enabled": true
+    },
+    {
+      "groupName": "@mswjs/http-middleware",
+      "matchPackageNames": ["@mswjs/http-middleware"],
+      "reviewers": ["team:kibana-cloud-security-posture"],
+      "matchBaseBranches": ["main"],
+      "labels": ["Team:Cloud Security", "release_note:skip", "backport:skip"],
+      "minimumReleaseAge": "7 days",
+      "enabled": true
     }
   ],
   "customManagers": [

src/dev/performance/run_performance_cli.ts

@@ -35,6 +35,19 @@ interface TestRunProps extends EsRunProps {
   kibanaInstallDir: string | undefined;
 }
 
+interface JourneyTargetGroups {
+  [key: string]: string[];
+}
+
+const journeyTargetGroups: JourneyTargetGroups = {
+  kibanaStartAndLoad: ['login'],
+  crud: ['tags_listing_page', 'dashboard_listing_page'],
+  dashboard: ['ecommerce_dashboard', 'data_stress_test_lens', 'flight_dashboard'],
+  discover: ['many_fields_discover', 'many_fields_discover_esql'],
+  maps: ['ecommerce_dashboard_map_only'],
+  ml: ['aiops_log_rate_analysis', 'many_fields_transform', 'tsdb_logs_data_visualizer'],
+};
+
 const readFilesRecursively = (dir: string, callback: Function) => {
   const files = fs.readdirSync(dir);
   files.forEach((file) => {
@@ -48,6 +61,44 @@ const readFilesRecursively = (dir: string, callback: Function) => {
   });
 };
 
+const getAllJourneys = (dir: string) => {
+  const journeys: Journey[] = [];
+
+  readFilesRecursively(dir, (filePath: string) =>
+    journeys.push({
+      name: path.parse(filePath).name,
+      path: path.resolve(dir, filePath),
+    })
+  );
+
+  return journeys;
+};
+
+const getJourneysToRun = ({ journeyPath, group }: { journeyPath?: string; group?: string }) => {
+  if (group && typeof group === 'string') {
+    if (!(group in journeyTargetGroups)) {
+      throw createFlagError(`Group '${group}' is not defined, try again`);
+    }
+
+    const fileNames = journeyTargetGroups[group];
+    const dir = path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
+
+    return getAllJourneys(dir).filter((journey) => fileNames.includes(journey.name));
+  }
+
+  if (journeyPath && !fs.existsSync(journeyPath)) {
+    throw createFlagError('--journey-path must be an existing path');
+  }
+
+  if (journeyPath && fs.statSync(journeyPath).isFile()) {
+    return [{ name: path.parse(journeyPath).name, path: journeyPath }];
+  } else {
+    // default dir is x-pack/performance/journeys_e2e
+    const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
+    return getAllJourneys(dir);
+  }
+};
+
 async function startEs(props: EsRunProps) {
   const { procRunner, log, logsDir } = props;
   await procRunner.run('es', {
@@ -115,29 +166,17 @@ run(
     const skipWarmup = flagsReader.boolean('skip-warmup');
     const kibanaInstallDir = flagsReader.path('kibana-install-dir');
     const journeyPath = flagsReader.path('journey-path');
+    const group = flagsReader.string('group');
 
-    if (kibanaInstallDir && !fs.existsSync(kibanaInstallDir)) {
-      throw createFlagError('--kibana-install-dir must be an existing directory');
+    if (group && journeyPath) {
+      throw createFlagError('--group and --journeyPath cannot be used simultaneously');
     }
 
-    if (journeyPath && !fs.existsSync(journeyPath)) {
-      throw createFlagError('--journey-path must be an existing path');
+    if (kibanaInstallDir && !fs.existsSync(kibanaInstallDir)) {
+      throw createFlagError('--kibana-install-dir must be an existing directory');
     }
 
-    const journeys: Journey[] = [];
-
-    if (journeyPath && fs.statSync(journeyPath).isFile()) {
-      journeys.push({ name: path.parse(journeyPath).name, path: journeyPath });
-    } else {
-      // default dir is x-pack/performance/journeys_e2e
-      const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
-      readFilesRecursively(dir, (filePath: string) =>
-        journeys.push({
-          name: path.parse(filePath).name,
-          path: path.resolve(dir, filePath),
-        })
-      );
-    }
+    const journeys = getJourneysToRun({ journeyPath, group });
 
     if (journeys.length === 0) {
       throw new Error('No journeys found');
@@ -191,13 +230,14 @@ run(
   },
   {
     flags: {
-      string: ['kibana-install-dir', 'journey-path'],
+      string: ['kibana-install-dir', 'journey-path', 'group'],
       boolean: ['skip-warmup'],
       help: `
       --kibana-install-dir=dir      Run Kibana from existing install directory instead of from source
       --journey-path=path           Define path to performance journey or directory with multiple journeys
                                     that should be executed. '${JOURNEY_BASE_PATH}' is run by default
       --skip-warmup                 Journey will be executed without warmup (TEST phase only)
+      --group                       Run subset of journeys, defined in the specified group
     `,
     },
   }

src/plugins/console/server/lib/spec_definitions/json/generated/esql.query.json

@@ -5,7 +5,16 @@
       "filter_path": [],
       "human": "__flag__",
       "pretty": "__flag__",
-      "format": "",
+      "format": [
+        "csv",
+        "json",
+        "tsv",
+        "txt",
+        "yaml",
+        "cbor",
+        "smile",
+        "arrow"
+      ],
       "delimiter": ""
     },
     "methods": [

src/plugins/console/server/lib/spec_definitions/json/generated/indices.resolve_index.json

@@ -11,7 +11,9 @@
         "closed",
         "hidden",
         "none"
-      ]
+      ],
+      "ignore_unavailable": "__flag__",
+      "allow_no_indices": "__flag__"
     },
     "methods": [
       "GET"

src/plugins/console/server/lib/spec_definitions/json/generated/ingest.delete_geoip_database.json

@@ -0,0 +1,31 @@
+{
+  "ingest.delete_geoip_database": {
+    "url_params": {
+      "error_trace": "__flag__",
+      "filter_path": [],
+      "human": "__flag__",
+      "pretty": "__flag__",
+      "master_timeout": [
+        "30s",
+        "-1",
+        "0"
+      ],
+      "timeout": [
+        "30s",
+        "-1",
+        "0"
+      ]
+    },
+    "methods": [
+      "DELETE"
+    ],
+    "patterns": [
+      "_ingest/geoip/database/{id}"
+    ],
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-geoip-database-api.html",
+    "availability": {
+      "stack": true,
+      "serverless": false
+    }
+  }
+}