[Discover] Allow to store the configured ES|QL visualization v3

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -134,7 +134,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "risk-engine-configuration": "b105d4a3c6adce40708d729d12e5ef3c8fbd9508",
         "rules-settings": "892a2918ebaeba809a612b8d97cec0b07c800b5f",
         "sample-data-telemetry": "37441b12f5b0159c2d6d5138a494c9f440e950b5",
-        "search": "2c1ab8a17e6972be2fa8d3880ba2305dfd9a5a6e",
+        "search": "04a2141f9fa9e59cf824ad23ae3982fb1556dea2",
         "search-session": "b2fcd840e12a45039ada50b1355faeafa39876d1",
         "search-telemetry": "b568601618744720b5662946d3103e3fb75fe8ee",
         "security-rule": "07abb4d7e707d91675ec0495c73816394c7b521f",

src/plugins/discover/public/application/main/components/layout/discover_histogram_layout.tsx

@@ -33,6 +33,7 @@ export const DiscoverHistogramLayout = ({
   const { dataState } = stateContainer;
   const searchSessionId = useObservable(stateContainer.searchSessionManager.searchSessionId$);
   const hideChart = useAppStateSelector((state) => state.hideChart);
+
   const unifiedHistogramProps = useDiscoverHistogram({
     stateContainer,
     inspectorAdapters: dataState.inspectorAdapters,

src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts

@@ -8,6 +8,7 @@
 
 import { useQuerySubscriber } from '@kbn/unified-field-list/src/hooks/use_query_subscriber';
 import {
+  ExternalVisContext,
   UnifiedHistogramApi,
   UnifiedHistogramFetchStatus,
   UnifiedHistogramState,
@@ -88,13 +89,17 @@ export const useDiscoverHistogram = ({
     const subscription = createUnifiedHistogramStateObservable(unifiedHistogram?.state$)?.subscribe(
       (changes) => {
         const { lensRequestAdapter, ...stateChanges } = changes;
+
         const appState = stateContainer.appState.getState();
         const oldState = {
           hideChart: appState.hideChart,
           interval: appState.interval,
           breakdownField: appState.breakdownField,
         };
-        const newState = { ...oldState, ...stateChanges };
+        const newState = {
+          ...oldState,
+          ...stateChanges,
+        };
 
         if ('lensRequestAdapter' in changes) {
           inspectorAdapters.lensRequests = lensRequestAdapter;
@@ -217,10 +222,12 @@ export const useDiscoverHistogram = ({
   const {
     dataView: textBasedDataView,
     query: textBasedQuery,
+    externalVisContext: textBasedExternalVisContext,
     columns,
   } = useObservable(textBasedFetchComplete$, {
     dataView: stateContainer.internalState.getState().dataView!,
-    query: stateContainer.appState.getState().query,
+    query: stateContainer.appState.getState().query!,
+    externalVisContext: stateContainer.appState.getState().visContext,
     columns: savedSearchData$.documents$.getValue().textBasedQueryColumns ?? [],
   });
 
@@ -229,12 +236,14 @@ export const useDiscoverHistogram = ({
       return;
     }
 
-    const fetchStart = stateContainer.dataState.fetch$.subscribe(() => {
+    const fetchStart = stateContainer.dataState.fetch$.subscribe((value) => {
+      // console.log('fetchStart', value);
       if (!skipRefetch.current) {
         setIsSuggestionLoading(true);
       }
     });
-    const fetchComplete = textBasedFetchComplete$.subscribe(() => {
+    const fetchComplete = textBasedFetchComplete$.subscribe((value) => {
+      // console.log('fetchComplete', value);
       setIsSuggestionLoading(false);
     });
 
@@ -321,6 +330,14 @@ export const useDiscoverHistogram = ({
   // eslint-disable-next-line react-hooks/exhaustive-deps
   const timeRangeMemoized = useMemo(() => timeRange, [timeRange?.from, timeRange?.to]);
 
+  const onVisContextChanged = useCallback(
+    (newVisContext: ExternalVisContext | undefined) => {
+      // console.log('got new vis context from histogram', newVisContext);
+      stateContainer.appState.update({ visContext: newVisContext });
+    },
+    [stateContainer]
+  );
+
   return {
     ref,
     getCreationOptions,
@@ -336,6 +353,8 @@ export const useDiscoverHistogram = ({
     withDefaultActions: histogramCustomization?.withDefaultActions,
     disabledActions: histogramCustomization?.disabledActions,
     isChartLoading: isSuggestionLoading,
+    externalVisContext: isPlainRecord ? textBasedExternalVisContext : undefined, // visContext should be in sync with current query
+    onVisContextChanged: isPlainRecord ? onVisContextChanged : undefined,
   };
 };
 
@@ -349,7 +368,9 @@ const createUnifiedHistogramStateObservable = (state$?: Observable<UnifiedHistog
     startWith(undefined),
     pairwise(),
     map(([prev, curr]) => {
-      const changes: Partial<DiscoverAppState> & { lensRequestAdapter?: RequestAdapter } = {};
+      const changes: Partial<DiscoverAppState> & {
+        lensRequestAdapter?: RequestAdapter;
+      } = {};
 
       if (!curr) {
         return changes;
@@ -413,6 +434,7 @@ const createFetchCompleteObservable = (stateContainer: DiscoverStateContainer) =
     map(({ textBasedQueryColumns }) => ({
       dataView: stateContainer.internalState.getState().dataView!,
       query: stateContainer.appState.getState().query!,
+      externalVisContext: stateContainer.appState.getState().visContext,
       columns: textBasedQueryColumns ?? [],
     }))
   );
@@ -427,7 +449,7 @@ const createTotalHitsObservable = (state$?: Observable<UnifiedHistogramState>) =
 
 const createCurrentSuggestionObservable = (state$: Observable<UnifiedHistogramState>) => {
   return state$.pipe(
-    map((state) => state.currentSuggestion),
+    map((state) => state.currentSuggestionContext),
     distinctUntilChanged(isEqual)
   );
 };

src/plugins/discover/public/application/main/components/top_nav/get_top_nav_links.tsx

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { omit } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { TopNavMenuData } from '@kbn/navigation-plugin/public';
@@ -138,7 +139,7 @@ export const getTopNavLinks = ({
 
       // Share -> Get links -> Snapshot
       const params: DiscoverAppLocatorParams = {
-        ...appState,
+        ...omit(appState, 'visContext'), // TODO: Should we make the lens vis sharable? Looks like it's not serializable.
         ...(savedSearch.id ? { savedSearchId: savedSearch.id } : {}),
         ...(dataView?.isPersisted()
           ? { dataViewId: dataView?.id }

src/plugins/discover/public/application/main/services/discover_app_state_container.ts

@@ -20,6 +20,7 @@ import {
   FilterStateStore,
   Query,
 } from '@kbn/es-query';
+import type { ExternalVisContext } from '@kbn/unified-histogram-plugin/public';
 import { SavedSearch, VIEW_MODE } from '@kbn/saved-search-plugin/public';
 import { IKbnUrlStateStorage, ISyncStateRef, syncState } from '@kbn/kibana-utils-plugin/public';
 import { isEqual } from 'lodash';
@@ -143,6 +144,10 @@ export interface DiscoverAppState {
    * Breakdown field of chart
    */
   breakdownField?: string;
+  /**
+   * Customization for Lens vis in Histogram area
+   */
+  visContext?: ExternalVisContext;
 }
 
 export const { Provider: DiscoverAppStateProvider, useSelector: useAppStateSelector } =
@@ -177,6 +182,15 @@ export const getDiscoverAppStateContainer = ({
     },
   };
 
+  const enhancedAppContainerFiltered = {
+    ...enhancedAppContainer,
+    // get: () => {
+    //   const state = { ...appStateContainer.getState() };
+    //   delete state.visContext; // don't sync visContext to URL as it can be very long
+    //   return state;
+    // },
+  };
+
   const hasChanged = () => {
     return !isEqualState(initialState, appStateContainer.getState());
   };
@@ -202,14 +216,15 @@ export const getDiscoverAppStateContainer = ({
   const replaceUrlState = async (newPartial: DiscoverAppState = {}, merge = true) => {
     addLog('[appState] replaceUrlState', { newPartial, merge });
     const state = merge ? { ...appStateContainer.getState(), ...newPartial } : newPartial;
+    // delete state.visContext; // don't sync visContext to URL as it can be very long
     await stateStorage.set(APP_STATE_URL_KEY, state, { replace: true });
   };
 
   const startAppStateUrlSync = () => {
     addLog('[appState] start syncing state with URL');
     return syncState({
       storageKey: APP_STATE_URL_KEY,
-      stateContainer: enhancedAppContainer,
+      stateContainer: enhancedAppContainerFiltered,
       stateStorage,
     });
   };

src/plugins/discover/public/application/main/services/discover_saved_search_container.ts

@@ -264,7 +264,9 @@ export function isEqualSavedSearch(savedSearchPrev: SavedSearch, savedSearchNext
       return false; // ignore when value was changed from `undefined` to `false` as it happens per app logic, not by a user action
     }
 
-    const isSame = isEqual(prevSavedSearch[key], nextSavedSearchWithoutSearchSource[key]);
+    const prevValue = getSavedSearchValueForComparison(prevSavedSearch, key);
+    const nextValue = getSavedSearchValueForComparison(nextSavedSearchWithoutSearchSource, key);
+    const isSame = isEqual(prevValue, nextValue);
 
     if (!isSame) {
       addLog('[savedSearch] difference between initial and changed version', {
@@ -330,3 +332,15 @@ function getSearchSourceFieldValueForComparison(
 
   return searchSource.getField(searchSourceFieldName);
 }
+
+function getSavedSearchValueForComparison(
+  savedSearch: Omit<SavedSearch, 'searchSource'>,
+  key: keyof Omit<SavedSearch, 'searchSource'>
+) {
+  if (key === 'visContextJSON' && savedSearch.visContextJSON) {
+    // the stringified JSON might have keys in a different order, so we parse it back into an object to compare
+    return JSON.parse(savedSearch.visContextJSON);
+  }
+
+  return savedSearch[key];
+}

src/plugins/discover/public/application/main/services/discover_state.test.ts

@@ -447,6 +447,7 @@ describe('Test discover state actions', () => {
         "sort": Array [],
         "timeRange": undefined,
         "usesAdHocDataView": false,
+        "visContextJSON": undefined,
       }
     `);
     expect(searchSource.getField('index')?.id).toEqual('the-data-view-id');

src/plugins/discover/public/application/main/utils/get_state_defaults.ts

@@ -9,7 +9,10 @@
 import { cloneDeep, isEqual } from 'lodash';
 import { IUiSettingsClient } from '@kbn/core/public';
 import { SavedSearch } from '@kbn/saved-search-plugin/public';
-import { getChartHidden } from '@kbn/unified-histogram-plugin/public';
+import {
+  getChartHidden,
+  fromExternalVisContextJSONString,
+} from '@kbn/unified-histogram-plugin/public';
 import {
   DEFAULT_COLUMNS_SETTING,
   DOC_HIDE_TIME_COLUMN_SETTING,
@@ -101,6 +104,9 @@ export function getStateDefaults({
   if (savedSearch.breakdownField) {
     defaultState.breakdownField = savedSearch.breakdownField;
   }
+  if (savedSearch.visContextJSON) {
+    defaultState.visContext = fromExternalVisContextJSONString(savedSearch.visContextJSON);
+  }
 
   return defaultState;
 }

src/plugins/discover/public/application/main/utils/update_saved_search.ts

@@ -7,6 +7,7 @@
  */
 import type { SavedSearch, SortOrder } from '@kbn/saved-search-plugin/public';
 import type { DataView } from '@kbn/data-views-plugin/common';
+import { toExternalVisContextJSONString } from '@kbn/unified-histogram-plugin/public';
 import { cloneDeep } from 'lodash';
 import { isTextBasedQuery } from './is_text_based_query';
 import type { DiscoverAppState } from '../services/discover_app_state_container';
@@ -64,6 +65,7 @@ export function updateSavedSearch({
     savedSearch.rowHeight = state.rowHeight;
     savedSearch.rowsPerPage = state.rowsPerPage;
     savedSearch.sampleSize = state.sampleSize;
+    savedSearch.visContextJSON = toExternalVisContextJSONString(state.visContext);
 
     if (state.viewMode) {
       savedSearch.viewMode = state.viewMode;

src/plugins/saved_search/common/content_management/v1/cm_services.ts

@@ -68,6 +68,7 @@ const savedSearchAttributesSchema = schema.object(
       })
     ),
     breakdownField: schema.maybe(schema.string()),
+    visContextJSON: schema.maybe(schema.string()), // TODO: should we create a new version?
     version: schema.maybe(schema.number()),
   },
   { unknowns: 'forbid' }

src/plugins/saved_search/common/saved_searches_utils.ts

@@ -34,4 +34,5 @@ export const fromSavedSearchAttributes = (
   rowsPerPage: attributes.rowsPerPage,
   sampleSize: attributes.sampleSize,
   breakdownField: attributes.breakdownField,
+  visContextJSON: attributes.visContextJSON,
 });

src/plugins/saved_search/common/service/get_saved_searches.test.ts

@@ -146,6 +146,7 @@ describe('getSavedSearch', () => {
         "title": "test1",
         "usesAdHocDataView": undefined,
         "viewMode": undefined,
+        "visContextJSON": undefined,
       }
     `);
   });
@@ -252,6 +253,7 @@ describe('getSavedSearch', () => {
         "title": "test2",
         "usesAdHocDataView": undefined,
         "viewMode": undefined,
+        "visContextJSON": undefined,
       }
     `);
   });

src/plugins/saved_search/common/service/saved_searches_utils.test.ts

@@ -88,6 +88,7 @@ describe('saved_searches_utils', () => {
           "title": "saved search",
           "usesAdHocDataView": false,
           "viewMode": undefined,
+          "visContextJSON": undefined,
         }
       `);
     });
@@ -138,6 +139,7 @@ describe('saved_searches_utils', () => {
           "title": "title",
           "usesAdHocDataView": false,
           "viewMode": undefined,
+          "visContextJSON": undefined,
         }
       `);
     });

src/plugins/saved_search/common/service/saved_searches_utils.ts

@@ -48,4 +48,5 @@ export const toSavedSearchAttributes = (
   rowsPerPage: savedSearch.rowsPerPage,
   sampleSize: savedSearch.sampleSize,
   breakdownField: savedSearch.breakdownField,
+  visContextJSON: savedSearch.visContextJSON,
 });

src/plugins/saved_search/common/types.ts

@@ -44,6 +44,7 @@ export interface SavedSearchAttributes {
   rowsPerPage?: number;
   sampleSize?: number;
   breakdownField?: string;
+  visContextJSON?: string;
 }
 
 /** @internal **/
@@ -74,6 +75,8 @@ export interface SavedSearch {
   rowsPerPage?: number;
   sampleSize?: number;
   breakdownField?: string;
+  visContextJSON?: string;
+
   references?: SavedObjectReference[];
   sharingSavedObjectProps?: {
     outcome?: SavedObjectsResolveResponse['outcome'];

src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts

@@ -240,6 +240,7 @@ describe('getSavedSearchAttributeService', () => {
           "title": "saved-search-title",
           "usesAdHocDataView": undefined,
           "viewMode": undefined,
+          "visContextJSON": undefined,
         }
       `);
     });

src/plugins/saved_search/server/content_management/saved_search_storage.ts

@@ -43,6 +43,7 @@ export class SavedSearchStorage extends SOContentStorage<SavedSearchCrudTypes> {
         'refreshInterval',
         'rowsPerPage',
         'breakdownField',
+        'visContextJSON',
         'sampleSize',
       ],
       logger,

src/plugins/saved_search/server/saved_objects/schema.ts

@@ -93,3 +93,7 @@ export const SCHEMA_SEARCH_V8_12_0 = schema.object({
     })
   ),
 });
+export const SCHEMA_SEARCH_V8_13_0 = schema.object({
+  ...SCHEMA_SEARCH_BASE,
+  visContextJSON: schema.maybe(schema.string()),
+});

src/plugins/saved_search/server/saved_objects/search.ts

@@ -10,7 +10,7 @@ import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common';
 import { getAllMigrations } from './search_migrations';
-import { SCHEMA_SEARCH_V8_8_0, SCHEMA_SEARCH_V8_12_0 } from './schema';
+import { SCHEMA_SEARCH_V8_8_0, SCHEMA_SEARCH_V8_12_0, SCHEMA_SEARCH_V8_13_0 } from './schema';
 
 export function getSavedSearchObjectType(
   getSearchSourceMigrations: () => MigrateFunctionsObject
@@ -45,7 +45,8 @@ export function getSavedSearchObjectType(
     schemas: {
       '8.8.0': SCHEMA_SEARCH_V8_8_0,
       '8.12.0': SCHEMA_SEARCH_V8_12_0,
+      '8.13.0': SCHEMA_SEARCH_V8_13_0,
     },
-    migrations: () => getAllMigrations(getSearchSourceMigrations()),
+    migrations: () => getAllMigrations(getSearchSourceMigrations()), // TODO: add lens embeddable migrations for `visContextJSON`
   };
 }

src/plugins/unified_histogram/public/__mocks__/data_view.ts

@@ -103,6 +103,7 @@ export const buildDataViewMock = ({
       return dataViewFields.find((field) => field.name === timeFieldName);
     },
     toSpec: () => ({}),
+    toMinimalSpec: () => ({}),
   } as unknown as DataView;
 
   dataView.isTimeBased = () => !!timeFieldName;