[Discover-Next] Adding datasources support for dataframes (#7106)

Parses user string between `::datasource::` to get the data source Then appends the dataSourceId in the meta.

Also creates the data frame before the interceptor to keep the meta info passed around.

If this gets accepted follow ups need to happen:
* should be ensuring the virtual index pattern includes the data source name with the index pattern name to avoid conflicts and a weird state.
* some reason dataframe is being always wiped out the first call in the search source. need to fix so that the schema is persisted
* weird initial load state. likely just too many things at once so we should make sure the usual and fresh experience is respected
* better indicator on the datasource, or just autocomplete that displays all the datasource connections when user types initial `::`.

Rebase of #7092 

Issue:
#6957

---------

Signed-off-by: Sean Li <lnse@amazon.com>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>

changelogs/fragments/7106.yml

@@ -0,0 +1,2 @@
+feat:
+- Onboard dataframes support to MDS and create dataframe before request ([#7106](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7106))

package.json

@@ -65,7 +65,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --data.enhancements.enabled=true --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['query:dataSource:readOnly']=false",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --data.enhancements.enabled=true --data_source.enabled=true --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['query:dataSource:readOnly']=false",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",

src/plugins/data/common/data_frames/utils.ts

@@ -45,6 +45,29 @@ export const getRawQueryString = (
   );
 };
 
+/**
+ * Parses a raw query string and extracts the query string and data source.
+ * @param rawQueryString - The raw query string to parse.
+ * @returns An object containing the parsed query string and data source (if found).
+ */
+export const parseRawQueryString = (rawQueryString: string) => {
+  const rawDataSource = rawQueryString.match(/::(.*?)::/);
+  return {
+    qs: rawQueryString.replace(/::.*?::/, ''),
+    formattedQs(key: string = '.'): string {
+      const parts = rawQueryString.split('::');
+      if (parts.length > 1) {
+        return (parts.slice(0, 1).join('') + parts.slice(1).join(key)).replace(
+          new RegExp(key + '$'),
+          ''
+        );
+      }
+      return rawQueryString;
+    },
+    ...(rawDataSource && { dataSource: rawDataSource[1] }),
+  };
+};
+
 /**
  * Returns the raw aggregations from the search request.
  *
@@ -379,6 +402,7 @@ export const updateDataFrameMeta = ({
   getAggQsFn: GetDataFrameAggQsFn;
 }) => {
   dataFrame.meta = {
+    ...dataFrame.meta,
     aggs: aggConfig,
     aggsQs: {
       [aggConfig.id]: getAggQsFn({

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -141,6 +141,25 @@ export class IndexPatternsService {
     return await this.savedObjectsClient.get<DataSourceAttributes>('data-source', id);
   };
 
+  /**
+   * Finds a data source by its title.
+   *
+   * @param title - The title of the data source to find.
+   * @param size - The number of results to return. Defaults to 10.
+   * @returns The first matching data source or undefined if not found.
+   */
+  findDataSourceByTitle = async (title: string, size: number = 10) => {
+    const savedObjectsResponse = await this.savedObjectsClient.find<DataSourceAttributes>({
+      type: 'data-source',
+      fields: ['title'],
+      search: title,
+      searchFields: ['title'],
+      perPage: size,
+    });
+
+    return savedObjectsResponse[0] || undefined;
+  };
+
   /**
    * Get list of index pattern ids
    * @param refresh Force refresh of index pattern list

src/plugins/data/common/search/search_source/search_source.ts

@@ -87,7 +87,15 @@ import { normalizeSortRequest } from './normalize_sort_request';
 import { filterDocvalueFields } from './filter_docvalue_fields';
 import { fieldWildcardFilter } from '../../../../opensearch_dashboards_utils/common';
 import { IIndexPattern } from '../../index_patterns';
-import { DATA_FRAME_TYPES, IDataFrame, IDataFrameResponse, convertResult } from '../../data_frames';
+import {
+  DATA_FRAME_TYPES,
+  IDataFrame,
+  IDataFrameResponse,
+  convertResult,
+  createDataFrame,
+  getRawQueryString,
+  parseRawQueryString,
+} from '../../data_frames';
 import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..';
 import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types';
 import { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types';
@@ -305,6 +313,23 @@ export class SearchSource {
     return this.getDataFrame();
   }
 
+  /**
+   * Create and set the data frame of this SearchSource
+   *
+   * @async
+   * @return {undefined|IDataFrame}
+   */
+  async createDataFrame(searchRequest: SearchRequest) {
+    const rawQueryString = this.getRawQueryStringFromRequest(searchRequest);
+    const dataFrame = createDataFrame({
+      name: searchRequest.index.title || searchRequest.index,
+      fields: [],
+      ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }),
+    });
+    await this.setDataFrame(dataFrame);
+    return this.getDataFrame();
+  }
+
   /**
    * Clear the data frame of this SearchSource
    */
@@ -401,6 +426,10 @@ export class SearchSource {
   private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) {
     const { search, getConfig, onResponse } = this.dependencies;
 
+    if (!this.getDataFrame()) {
+      await this.createDataFrame(searchRequest);
+    }
+
     const params = getExternalSearchParamsFromRequest(searchRequest, {
       getConfig,
       getDataFrame: this.getDataFrame.bind(this),
@@ -444,6 +473,10 @@ export class SearchSource {
     return request.body!.query.hasOwnProperty('type') && request.body!.query.type === 'unsupported';
   }
 
+  private getRawQueryStringFromRequest(request: SearchRequest): string | undefined {
+    return getRawQueryString({ params: request });
+  }
+
   /**
    *  Called by requests of this search source when they are started
    *  @param options

src/plugins/data/public/search/search_service.ts

@@ -145,6 +145,16 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
         if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) {
           indexPatterns.clearCache(this.dfCache.get()!.name, false);
         }
+        if (
+          dataFrame.meta &&
+          dataFrame.meta.queryConfig &&
+          'dataSource' in dataFrame.meta.queryConfig
+        ) {
+          const dataSource = await indexPatterns.findDataSourceByTitle(
+            dataFrame.meta.queryConfig.dataSource
+          );
+          dataFrame.meta.queryConfig.dataSourceId = dataSource?.id;
+        }
         this.dfCache.set(dataFrame);
         const existingIndexPattern = indexPatterns.getByTitle(dataFrame.name!, true);
         const dataSet = await indexPatterns.create(

src/plugins/data/server/search/search_service.ts

@@ -218,6 +218,16 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
               if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) {
                 scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false);
               }
+              if (
+                dataFrame.meta &&
+                dataFrame.meta.queryConfig &&
+                'dataSource' in dataFrame.meta.queryConfig
+              ) {
+                const dataSource = await scopedIndexPatterns.findDataSourceByTitle(
+                  dataFrame.meta.queryConfig.dataSource
+                );
+                dataFrame.meta.queryConfig.dataSourceId = dataSource?.id;
+              }
               this.dfCache.set(dataFrame);
               const existingIndexPattern = scopedIndexPatterns.getByTitle(dataFrame.name!, true);
               const dataSet = await scopedIndexPatterns.create(