Merge branch 'main' into kbn-104081-split-multiple-indices-logic

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -76,19 +76,19 @@ buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image-aarch64.tar.g
 buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"
 cd -
 
+# This part is related with updating the configuration of kibana-controller,
+# so that new stack instances contain the latest and greatest image of kibana,
+# and the respective stack components of course.
 echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
-
   cat << EOF | buildkite-agent pipeline upload
 steps:
-  - trigger: k8s-gitops-update-image-tag
+  - trigger: serverless-gitops-update-stack-image-tag
     async: true
     label: ":argo: Update image tag for Kibana"
     branches: main
     build:
       env:
-        MODE: sed
-        TARGET_FILE: kibana-controller.yaml
         IMAGE_TAG: "git-$GIT_ABBREV_COMMIT"
         SERVICE: kibana-controller
         NAMESPACE: kibana-ci

.github/CODEOWNERS

@@ -459,6 +459,7 @@ x-pack/packages/ml/local_storage @elastic/ml-ui
 x-pack/packages/ml/nested_property @elastic/ml-ui
 x-pack/plugins/ml @elastic/ml-ui
 x-pack/packages/ml/query_utils @elastic/ml-ui
+x-pack/packages/ml/random_sampler_utils @elastic/ml-ui
 x-pack/packages/ml/route_utils @elastic/ml-ui
 x-pack/packages/ml/string_hash @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui

docs/settings/apm-settings.asciidoc

@@ -101,4 +101,7 @@ Matcher for all source map indices. Defaults to `apm-*`.
 `xpack.apm.autoCreateApmDataView` {ess-icon}::
 Set to `false` to disable the automatic creation of the APM data view when the APM app is opened. Defaults to `true`.
 
+`xpack.apm.latestAgentVersionsUrl` {ess-icon}::
+Specifies the URL of a self hosted file that contains latest agent versions. Defaults to `https://apm-agent-versions.elastic.co/versions.json`. Set to `''` to disable requesting latest agent versions.
+
 // end::general-apm-settings[]

package.json

@@ -474,6 +474,7 @@
     "@kbn/ml-nested-property": "link:x-pack/packages/ml/nested_property",
     "@kbn/ml-plugin": "link:x-pack/plugins/ml",
     "@kbn/ml-query-utils": "link:x-pack/packages/ml/query_utils",
+    "@kbn/ml-random-sampler-utils": "link:x-pack/packages/ml/random_sampler_utils",
     "@kbn/ml-route-utils": "link:x-pack/packages/ml/route_utils",
     "@kbn/ml-string-hash": "link:x-pack/packages/ml/string_hash",
     "@kbn/ml-trained-models-utils": "link:x-pack/packages/ml/trained_models_utils",

packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts

@@ -141,6 +141,8 @@ export type ApmFields = Fields<{
       values: number[];
       counts: number[];
     };
+    'transaction.result': string;
+    'transaction.sampled': boolean;
     'service.environment': string;
     'service.framework.name': string;
     'service.framework.version': string;
@@ -164,8 +166,6 @@ export type ApmFields = Fields<{
     'span.self_time.sum.us': number;
     'span.subtype': string;
     'span.type': string;
-    'transaction.result': string;
-    'transaction.sampled': true;
     'span.links': Array<{
       trace: { id: string };
       span: { id: string };

packages/kbn-apm-synthtrace-client/src/lib/apm/transaction.ts

@@ -32,6 +32,7 @@ export class Transaction extends BaseSpan {
       error.fields['trace.id'] = this.fields['trace.id'];
       error.fields['transaction.id'] = this.fields['transaction.id'];
       error.fields['transaction.type'] = this.fields['transaction.type'];
+      error.fields['transaction.sampled'] = this.fields['transaction.sampled'];
     });
 
     return this;
@@ -43,6 +44,7 @@ export class Transaction extends BaseSpan {
       error.fields['transaction.id'] = this.fields['transaction.id'];
       error.fields['transaction.name'] = this.fields['transaction.name'];
       error.fields['transaction.type'] = this.fields['transaction.type'];
+      error.fields['transaction.sampled'] = this.fields['transaction.sampled'];
     });
 
     this._errors.push(...errors);
@@ -56,7 +58,10 @@ export class Transaction extends BaseSpan {
   }
 
   sample(sampled: boolean = true) {
-    this._sampled = sampled;
+    this._sampled = this.fields['transaction.sampled'] = sampled;
+    this._errors.forEach((error) => {
+      error.fields['transaction.sampled'] = sampled;
+    });
     return this;
   }
 

packages/kbn-doc-links/src/get_doc_links.ts

@@ -137,6 +137,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => {
       crawlerOverview: `${ENTERPRISE_SEARCH_DOCS}crawler.html`,
       deployTrainedModels: `${MACHINE_LEARNING_DOCS}ml-nlp-deploy-models.html`,
       documentLevelSecurity: `${ELASTICSEARCH_DOCS}document-level-security.html`,
+      elser: `${MACHINE_LEARNING_DOCS}ml-nlp-elser.html`,
       engines: `${ENTERPRISE_SEARCH_DOCS}engines.html`,
       ingestPipelines: `${ENTERPRISE_SEARCH_DOCS}ingest-pipelines.html`,
       languageAnalyzers: `${ELASTICSEARCH_DOCS}analysis-lang-analyzer.html`,

packages/kbn-doc-links/src/types.ts

@@ -122,6 +122,7 @@ export interface DocLinks {
     readonly crawlerOverview: string;
     readonly deployTrainedModels: string;
     readonly documentLevelSecurity: string;
+    readonly elser: string;
     readonly engines: string;
     readonly ingestPipelines: string;
     readonly languageAnalyzers: string;

packages/kbn-slo-schema/src/rest_specs/slo.ts

@@ -54,7 +54,7 @@ const getSLOParamsSchema = t.type({
 });
 
 const sortDirectionSchema = t.union([t.literal('asc'), t.literal('desc')]);
-const sortBySchema = t.union([t.literal('name'), t.literal('indicatorType')]);
+const sortBySchema = t.union([t.literal('creationTime'), t.literal('indicatorType')]);
 
 const findSLOParamsSchema = t.partial({
   query: t.partial({

packages/kbn-slo-schema/src/schema/indicators.ts

@@ -50,17 +50,13 @@ const apmTransactionErrorRateIndicatorSchema = t.type({
 const kqlCustomIndicatorTypeSchema = t.literal('sli.kql.custom');
 const kqlCustomIndicatorSchema = t.type({
   type: kqlCustomIndicatorTypeSchema,
-  params: t.intersection([
-    t.type({
-      index: t.string,
-      filter: t.string,
-      good: t.string,
-      total: t.string,
-    }),
-    t.partial({
-      timestampField: t.string,
-    }),
-  ]),
+  params: t.type({
+    index: t.string,
+    filter: t.string,
+    good: t.string,
+    total: t.string,
+    timestampField: t.string,
+  }),
 });
 
 const indicatorDataSchema = t.type({

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -57,7 +57,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
       Object {
         "action": "6cfc277ed3211639e37546ac625f4a68f2494215",
         "action_task_params": "5f419caba96dd8c77d0f94013e71d43890e3d5d6",
-        "alert": "1e4cd6941f1eb39c729c646e91fbfb9700de84b9",
+        "alert": "7bec97d7775a025ecf36a33baf17386b9e7b4c3c",
         "api_key_pending_invalidation": "16e7bcf8e78764102d7f525542d5b616809a21ee",
         "apm-indices": "d19dd7fb51f2d2cbc1f8769481721e0953f9a6d2",
         "apm-server-schema": "1d42f17eff9ec6c16d3a9324d9539e2d123d0a9a",

src/plugins/dashboard/public/dashboard_app/top_nav/controls_toolbar_button/controls_toolbar_button.tsx

@@ -8,7 +8,7 @@
 
 import React from 'react';
 
-import { EuiContextMenuPanel } from '@elastic/eui';
+import { EuiContextMenuPanel, useEuiTheme } from '@elastic/eui';
 import { ToolbarPopover } from '@kbn/shared-ux-button-toolbar';
 import type { ControlGroupContainer } from '@kbn/controls-plugin/public';
 
@@ -18,11 +18,15 @@ import { AddTimeSliderControlButton } from './add_time_slider_control_button';
 import { EditControlGroupButton } from './edit_control_group_button';
 
 export function ControlsToolbarButton({ controlGroup }: { controlGroup: ControlGroupContainer }) {
+  const { euiTheme } = useEuiTheme();
+
   return (
     <ToolbarPopover
       ownFocus
-      label={getControlButtonTitle()}
+      repositionOnScroll
       panelPaddingSize="none"
+      label={getControlButtonTitle()}
+      zIndex={Number(euiTheme.levels.header) - 1}
       data-test-subj="dashboard-controls-menu-button"
     >
       {({ closePopover }: { closePopover: () => void }) => (

src/plugins/dashboard/public/dashboard_app/top_nav/editor_menu.tsx

@@ -14,6 +14,7 @@ import {
   EuiContextMenuPanelItemDescriptor,
   EuiFlexGroup,
   EuiFlexItem,
+  useEuiTheme,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { ToolbarPopover } from '@kbn/shared-ux-button-toolbar';
@@ -53,6 +54,8 @@ export const EditorMenu = ({ createNewVisType, createNewEmbeddable }: Props) =>
     },
   } = pluginServices.getServices();
 
+  const { euiTheme } = useEuiTheme();
+
   const embeddableFactories = useMemo(
     () => Array.from(embeddable.getEmbeddableFactories()),
     [embeddable]
@@ -262,6 +265,8 @@ export const EditorMenu = ({ createNewVisType, createNewEmbeddable }: Props) =>
   };
   return (
     <ToolbarPopover
+      zIndex={Number(euiTheme.levels.header) - 1}
+      repositionOnScroll
       ownFocus
       label={i18n.translate('dashboard.solutionToolbar.editorMenuButtonLabel', {
         defaultMessage: 'Select type',

src/plugins/unified_search/public/__stories__/search_bar.stories.tsx

@@ -8,11 +8,12 @@
 
 import React from 'react';
 import { action } from '@storybook/addon-actions';
-import type { Query } from '@kbn/es-query';
+import type { DataViewBase, Query } from '@kbn/es-query';
 import { storiesOf } from '@storybook/react';
 import { I18nProvider } from '@kbn/i18n-react';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public';
+import { buildExistsFilter } from '@kbn/es-query';
 import { SearchBar, SearchBarProps } from '../search_bar';
 import { setIndexPatterns } from '../services';
 
@@ -260,6 +261,15 @@ storiesOf('SearchBar', module)
       showQueryInput: false,
     } as SearchBarProps)
   )
+  .add('with additional filters used for suggestions', () =>
+    wrapSearchBarInContext({
+      filtersForSuggestions: [
+        buildExistsFilter({ type: 'keyword', name: 'geo.src' }, {
+          id: undefined,
+        } as unknown as DataViewBase),
+      ],
+    } as unknown as SearchBarProps)
+  )
   .add('with only the filter bar on', () =>
     wrapSearchBarInContext({
       showDatePicker: false,

src/plugins/unified_search/public/filter_bar/filter_bar.tsx

@@ -22,6 +22,7 @@ export interface Props {
   indexPatterns: DataView[];
   intl: InjectedIntl;
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
   hiddenPanelOptions?: FilterItemsProps['hiddenPanelOptions'];
   /**
    * Applies extra styles necessary when coupled with the query bar
@@ -54,6 +55,7 @@ const FilterBarUI = React.memo(function FilterBarUI(props: Props) {
         onFiltersUpdated={props.onFiltersUpdated}
         indexPatterns={props.indexPatterns!}
         timeRangeForSuggestionsOverride={props.timeRangeForSuggestionsOverride}
+        filtersForSuggestions={props.filtersForSuggestions}
         hiddenPanelOptions={props.hiddenPanelOptions}
         readOnly={props.isDisabled}
       />

src/plugins/unified_search/public/filter_bar/filter_editor/filter_editor.tsx

@@ -126,6 +126,7 @@ export interface FilterEditorComponentProps {
   onLocalFilterCreate?: (initialState: { filter: Filter; queryDslFilter: QueryDslFilter }) => void;
   onLocalFilterUpdate?: (filter: Filter | QueryDslFilter) => void;
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
   mode?: 'edit' | 'add';
 }
 
@@ -334,6 +335,7 @@ class FilterEditorComponent extends Component<FilterEditorProps, State> {
             <FiltersBuilder
               filters={[localFilter]}
               timeRangeForSuggestionsOverride={this.props.timeRangeForSuggestionsOverride}
+              filtersForSuggestions={this.props.filtersForSuggestions}
               dataView={selectedDataView!}
               onChange={this.onLocalFilterChange}
               disabled={!selectedDataView}

src/plugins/unified_search/public/filter_bar/filter_editor/phrase_suggestor.tsx

@@ -12,13 +12,15 @@ import { UI_SETTINGS } from '@kbn/data-plugin/common';
 import { DataView, DataViewField } from '@kbn/data-views-plugin/common';
 import { debounce } from 'lodash';
 
+import { buildQueryFromFilters, Filter } from '@kbn/es-query';
 import { IUnifiedSearchPluginServices } from '../../types';
 
 export interface PhraseSuggestorProps {
   kibana: KibanaReactContextValue<IUnifiedSearchPluginServices>;
   indexPattern: DataView;
   field: DataViewField;
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
 }
 
 export interface PhraseSuggestorState {
@@ -73,7 +75,7 @@ export class PhraseSuggestorUI<T extends PhraseSuggestorProps> extends React.Com
   protected updateSuggestions = debounce(async (query: string = '') => {
     if (this.abortController) this.abortController.abort();
     this.abortController = new AbortController();
-    const { indexPattern, field, timeRangeForSuggestionsOverride } = this
+    const { indexPattern, field, timeRangeForSuggestionsOverride, filtersForSuggestions } = this
       .props as PhraseSuggestorProps;
     if (!field || !this.isSuggestingValues()) {
       return;
@@ -85,6 +87,8 @@ export class PhraseSuggestorUI<T extends PhraseSuggestorProps> extends React.Com
       query,
       signal: this.abortController.signal,
       useTimeRange: timeRangeForSuggestionsOverride,
+      boolFilter: buildQueryFromFilters(filtersForSuggestions, undefined).filter,
+      method: filtersForSuggestions?.length ? 'terms_agg' : 'terms_enum',
     });
 
     this.setState({ suggestions, isLoading: false });

src/plugins/unified_search/public/filter_bar/filter_item/filter_item.tsx

@@ -56,6 +56,7 @@ export interface FilterItemProps extends WithCloseFilterEditorConfirmModalProps
   uiSettings: IUiSettingsClient;
   hiddenPanelOptions?: FilterPanelOption[];
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
   readOnly?: boolean;
 }
 
@@ -391,6 +392,7 @@ function FilterItemComponent(props: FilterItemProps) {
                 onLocalFilterCreate={onLocalFilterCreate}
                 onCancel={() => setIsPopoverOpen(false)}
                 timeRangeForSuggestionsOverride={props.timeRangeForSuggestionsOverride}
+                filtersForSuggestions={props.filtersForSuggestions}
               />
             </div>,
           ]}

src/plugins/unified_search/public/filter_bar/filter_item/filter_items.tsx

@@ -35,6 +35,8 @@ export interface FilterItemsProps {
   intl: InjectedIntl;
   /** Controls whether or not filter suggestions are influenced by the global time */
   timeRangeForSuggestionsOverride?: boolean;
+  /** adds additional filters to be used for suggestions */
+  filtersForSuggestions?: Filter[];
   /** Array of panel options that controls the styling of each filter pill */
   hiddenPanelOptions?: FilterItemProps['hiddenPanelOptions'];
 }
@@ -74,6 +76,7 @@ const FilterItemsUI = React.memo(function FilterItemsUI(props: FilterItemsProps)
           uiSettings={uiSettings!}
           hiddenPanelOptions={props.hiddenPanelOptions}
           timeRangeForSuggestionsOverride={props.timeRangeForSuggestionsOverride}
+          filtersForSuggestions={props.filtersForSuggestions}
           readOnly={readOnly}
         />
       </EuiFlexItem>

src/plugins/unified_search/public/filters_builder/context.ts

@@ -8,6 +8,7 @@
 
 import React, { Dispatch } from 'react';
 import type { DataView } from '@kbn/data-views-plugin/common';
+import { Filter } from '@kbn/es-query';
 import type { FiltersBuilderActions } from './reducer';
 
 interface FiltersBuilderContextType {
@@ -19,6 +20,7 @@ interface FiltersBuilderContextType {
   };
   dropTarget: string;
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
   disabled: boolean;
 }
 

src/plugins/unified_search/public/filters_builder/filter_item/filter_item.tsx

@@ -96,6 +96,7 @@ export function FilterItem({
     dropTarget,
     globalParams: { hideOr },
     timeRangeForSuggestionsOverride,
+    filtersForSuggestions,
     disabled,
   } = useContext(FiltersBuilderContextType);
   const conditionalOperationType = getBooleanRelationType(filter);
@@ -309,6 +310,7 @@ export function FilterItem({
                                   onHandleParamsChange={onHandleParamsChange}
                                   onHandleParamsUpdate={onHandleParamsUpdate}
                                   timeRangeForSuggestionsOverride={timeRangeForSuggestionsOverride}
+                                  filtersForSuggestions={filtersForSuggestions}
                                 />
                               </div>
                             </EuiFormRow>

src/plugins/unified_search/public/filters_builder/filter_item/params_editor.tsx

@@ -21,6 +21,7 @@ interface ParamsEditorProps {
   onHandleParamsChange: (params: Filter['meta']['params']) => void;
   onHandleParamsUpdate: (value: string) => void;
   timeRangeForSuggestionsOverride?: boolean;
+  filtersForSuggestions?: Filter[];
   field?: DataViewField;
   operator?: Operator;
 }
@@ -33,6 +34,7 @@ export function ParamsEditor({
   onHandleParamsChange,
   onHandleParamsUpdate,
   timeRangeForSuggestionsOverride,
+  filtersForSuggestions,
 }: ParamsEditorProps) {
   const { disabled } = useContext(FiltersBuilderContextType);
   const onParamsChange = useCallback(
@@ -67,6 +69,7 @@ export function ParamsEditor({
           onParamsChange={onParamsChange}
           onParamsUpdate={onParamsUpdate}
           timeRangeForSuggestionsOverride={timeRangeForSuggestionsOverride}
+          filtersForSuggestions={filtersForSuggestions}
         />
       </EuiToolTip>
     </EuiFormRow>