From 62b0238dcbf71e5a830d87befacf07eb63f7b4df Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:14:24 -0700 Subject: [PATCH] [dataset-nav][bug] get connections by cluster and update client namespace (#7609) (#7611) * changes from abby * refactor api * polling db not populating * fixing datasource cache * fixing session id stuff and running async queries * Changeset file for PR #7609 created/updated --------- (cherry picked from commit d09895a134d92904eb540778796b63aaa6ee6a2f) Signed-off-by: Kawika Avilla Signed-off-by: abbyhu2000 Signed-off-by: Sean Li Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: abbyhu2000 Co-authored-by: Sean Li Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> --- changelogs/fragments/7609.yml | 2 + src/plugins/data/public/index.ts | 3 - .../dataset_navigator/dataset_navigator.tsx | 59 ++---- .../public/ui/dataset_navigator/index.tsx | 1 - .../lib/catalog_cache/cache_loader.tsx | 45 +++-- .../lib/catalog_cache/cache_manager.ts | 86 +++----- .../ui/dataset_navigator/lib/constants.ts | 190 ++++++++++-------- .../public/ui/dataset_navigator/lib/index.tsx | 1 - .../dataset_navigator/lib/requests/index.tsx | 6 - .../ui/dataset_navigator/lib/requests/sql.ts | 60 ------ .../public/ui/dataset_navigator/lib/types.tsx | 2 +- .../lib/utils/fetch_external_data_sources.ts | 40 ++-- .../ui/dataset_navigator/lib/utils/index.ts | 1 - .../lib/utils/query_session_utils.ts | 25 --- .../ui/dataset_navigator/lib/utils/shared.ts | 7 +- src/plugins/data/public/ui/index.ts | 7 +- .../data/public/ui/settings/settings.ts | 16 ++ .../query_enhancements/common/constants.ts | 2 + .../public/search/sql_search_interceptor.ts | 13 +- .../query_enhancements/server/index.ts | 9 +- .../query_enhancements/server/plugin.ts | 14 +- .../routes/data_source_connection/routes.ts | 78 ++++++- .../query_enhancements/server/routes/index.ts | 3 +- .../query_enhancements/server/search/index.ts | 1 + .../server/search/ppl_raw_search_strategy.ts | 4 +- .../server/search/ppl_search_strategy.ts | 2 +- .../search/sql_async_search_strategy.ts | 15 +- .../server/search/sql_search_strategy.ts | 2 +- .../query_enhancements/server/types.ts | 5 +- .../query_enhancements/server/utils/facet.ts | 16 +- .../query_enhancements/server/utils/index.ts | 2 +- .../server/utils/plugins.ts | 43 ++-- 32 files changed, 374 insertions(+), 386 deletions(-) create mode 100644 changelogs/fragments/7609.yml delete mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx delete mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts delete mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts diff --git a/changelogs/fragments/7609.yml b/changelogs/fragments/7609.yml new file mode 100644 index 000000000000..edb1d5a60c64 --- /dev/null +++ b/changelogs/fragments/7609.yml @@ -0,0 +1,2 @@ +fix: +- Dataset nav to load external connections and update namespace ([#7609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7609)) \ No newline at end of file diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index e8a64a0bcb6a..948bebed11ba 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -446,9 +446,6 @@ export { // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, DataSetNavigator, - setAsyncSessionId, - getAsyncSessionId, - setAsyncSessionIdByObj, } from './ui'; /** diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index bfeaad00bd4e..4912abd92614 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -40,13 +40,7 @@ import { getSearchService, getUiService, } from '../../services'; -import { - fetchDataSources, - fetchIndexPatterns, - fetchIndices, - isCatalogCacheFetching, - fetchIfExternalDataSourcesEnabled, -} from './lib'; +import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib'; import { useDataSetManager } from '../search_bar/lib/use_dataset_manager'; import { DataSetContract } from '../../query'; @@ -76,7 +70,6 @@ export const DataSetNavigator: React.FC = ({ const isInitialized = useRef(false); const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); - const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); const [dataSources, setDataSources] = useState([]); const [externalDataSources, setExternalDataSources] = useState([]); const [indexPatterns, setIndexPatterns] = useState([]); @@ -85,6 +78,7 @@ export const DataSetNavigator: React.FC = ({ const [selectedDataSetState, setSelectedDataSetState] = useState< SelectedDataSetState | undefined >(undefined); + const isExternalDataSourcesEnabled = externalDataSources.length > 0; const { loadStatus: dataSourcesLoadStatus, @@ -104,7 +98,7 @@ export const DataSetNavigator: React.FC = ({ const onRefresh = useCallback(() => { if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) { - startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); + startLoadingDataSources(dataSources); } }, [dataSourcesLoadStatus, dataSources, startLoadingDataSources]); @@ -158,19 +152,27 @@ export const DataSetNavigator: React.FC = ({ setIsLoading(true); try { - const [ - fetchedIndexPatterns, - fetchedDataSources, - fetchedIsExternalDataSourcesEnabled, - ] = await Promise.all([ + const [fetchedIndexPatterns, fetchedDataSources] = await Promise.all([ fetchIndexPatterns(savedObjectsClient!, ''), fetchDataSources(savedObjectsClient!), - fetchIfExternalDataSourcesEnabled(http!), ]); + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { + setExternalDataSources( + externalDataSourcesCache.dataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })) + ); + } else if (fetchedDataSources.length > 0) { + setExternalDataSources(await startLoadingDataSources(fetchedDataSources)); + } + setIndexPatterns(fetchedIndexPatterns); setDataSources(fetchedDataSources); - setIsExternalDataSourcesEnabled(fetchedIsExternalDataSourcesEnabled); + if (dataSet) { setSelectedDataSetState({ id: dataSet.id, @@ -206,7 +208,7 @@ export const DataSetNavigator: React.FC = ({ const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); if (status === DirectQueryLoadingStatus.SUCCESS) { setExternalDataSources( - externalDataSourcesCache.externalDataSources.map((ds) => ({ + externalDataSourcesCache.dataSources.map((ds) => ({ id: ds.dataSourceRef, name: ds.name, type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, @@ -462,25 +464,7 @@ export const DataSetNavigator: React.FC = ({ { name: S3DataSourcesLabel, panel: 4, - onClick: () => { - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if ( - (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || - externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(dataSourcesLoadStatus) && - dataSources.length > 0 - ) { - startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); - } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { - setExternalDataSources( - externalDataSourcesCache.externalDataSources.map((ds) => ({ - id: ds.dataSourceRef, - name: ds.name, - type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, - })) - ); - } - }, + onClick: () => {}, }, ] : []), @@ -527,7 +511,7 @@ export const DataSetNavigator: React.FC = ({ ), items: externalDataSources.map((dataSource) => ({ name: dataSource.name, - onClick: () => handleSelectExternalDataSource(dataSource), + onClick: async () => await handleSelectExternalDataSource(dataSource), panel: 5, })), content: isCatalogCacheFetching(dataSourcesLoadStatus) && createLoadingSpinner(), @@ -652,7 +636,6 @@ export const DataSetNavigator: React.FC = ({ databasesLoadStatus, cachedTables, tablesLoadStatus, - startLoadingDataSources, handleSelectedDataSet, handleSelectedDataSource, handleSelectedObject, diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index 3167afad74d9..e98e52c8421f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -5,4 +5,3 @@ export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; export { createDataSetNavigator } from './create_dataset_navigator'; -export { setAsyncSessionId, getAsyncSessionId, setAsyncSessionIdByObj } from './lib'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx index bae33f99a128..f5a6aa8aece0 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef, useState } from 'react'; import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; -import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; +import { ASYNC_QUERY, SPARK } from '../constants'; import { AsyncPollingResult, CachedColumn, @@ -16,12 +16,12 @@ import { DirectQueryLoadingStatus, DirectQueryRequest, } from '../types'; -import { getAsyncSessionId, setAsyncSessionIdByObj } from '../utils/query_session_utils'; +import { SIMPLE_DATA_SOURCE_TYPES, SimpleDataSource } from '../../../../../common'; import { addBackticksIfNeeded, combineSchemaAndDatarows, formatError } from '../utils/shared'; import { usePolling } from '../utils/use_polling'; -import { SQLService } from '../requests/sql'; import { CatalogCacheManager } from './cache_manager'; import { fetchExternalDataSources } from '../utils'; +import { getUiService } from '../../../../services'; export const updateDatabasesToCache = ( dataSourceName: string, @@ -96,10 +96,9 @@ export const updateTablesToCache = ( ); return; } - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); const newTables = combinedData - .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) + .filter((row: any) => !SPARK.HIVE_TABLE_REGEX.test(row.information)) .map((row: any) => ({ name: row.tableName, })); @@ -184,7 +183,7 @@ export const updateTableColumnsToCache = ( const tableColumns: CachedColumn[] = []; for (const row of combinedData) { - if (row.col_name === SPARK_PARTITION_INFO) { + if (row.col_name === SPARK.PARTITION_INFO) { break; } tableColumns.push({ @@ -282,7 +281,6 @@ export const useLoadToCache = ( http: HttpStart, notifications: NotificationsStart ) => { - const sqlService = new SQLService(http); const [currentDataSourceName, setCurrentDataSourceName] = useState(''); const [currentDatabaseName, setCurrentDatabaseName] = useState(''); const [currentTableName, setCurrentTableName] = useState(''); @@ -298,8 +296,13 @@ export const useLoadToCache = ( startPolling, stopPolling: stopLoading, } = usePolling((params) => { - return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current); - }, ASYNC_POLLING_INTERVAL); + return http.fetch(`../../api/enhancements/datasource/jobs`, { + query: { + id: dataSourceMDSClientId.current, + queryId: params.queryId, + }, + }); + }, ASYNC_QUERY.POLLING_INTERVAL); const onLoadingFailed = () => { setLoadStatus(DirectQueryLoadingStatus.FAILED); @@ -319,6 +322,7 @@ export const useLoadToCache = ( databaseName, tableName, }: StartLoadingParams) => { + const uiService = getUiService(); setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); setCurrentDataSourceName(dataSourceName); setCurrentDatabaseName(databaseName); @@ -331,14 +335,19 @@ export const useLoadToCache = ( datasource: dataSourceName, }; - const sessionId = getAsyncSessionId(dataSourceName); + const sessionId = uiService.Settings.getUserQuerySessionId(dataSourceName); if (sessionId) { requestPayload = { ...requestPayload, sessionId }; } - await sqlService - .fetch(requestPayload, dataSourceMDSId) + await http + .post(`../../api/enhancements/datasource/jobs`, { + body: JSON.stringify(requestPayload), + query: { + id: dataSourceMDSClientId.current, + }, + }) .then((result) => { - setAsyncSessionIdByObj(dataSourceName, result); + uiService.Settings.setUserQuerySessionIdByObj(dataSourceName, result); if (result.queryId) { startPolling({ queryId: result.queryId, @@ -443,7 +452,9 @@ export const useLoadExternalDataSourcesToCache = ( DirectQueryLoadingStatus.INITIAL ); - const loadExternalDataSources = async (connectedClusters: string[]) => { + const loadExternalDataSources = async ( + connectedClusters: SimpleDataSource[] + ): Promise => { setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); @@ -452,6 +463,11 @@ export const useLoadExternalDataSourcesToCache = ( CatalogCacheManager.updateExternalDataSources(externalDataSources); setLoadStatus(DirectQueryLoadingStatus.SUCCESS); CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); + return externalDataSources.map((dataSource) => ({ + id: dataSource.dataSourceRef, + name: dataSource.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })); } catch (error) { setLoadStatus(DirectQueryLoadingStatus.FAILED); CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); @@ -459,6 +475,7 @@ export const useLoadExternalDataSourcesToCache = ( title: 'Failed to load external datasources', }); } + return []; }; return { loadStatus, loadExternalDataSources }; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts index 3d0a8e0c982d..5496a8066c06 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts @@ -3,19 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { - ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE, - CATALOG_CACHE_VERSION, - RECENT_DATASET_OPTIONS_CACHE, -} from '../constants'; -import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; +import { ASYNC_QUERY, DATASET } from '../constants'; import { AccelerationsCacheData, CachedAccelerationByDataSource, CachedDataSource, CachedDataSourceStatus, CachedDatabase, - DataSetOption, DataSourceCacheData, ExternalDataSource, ExternalDataSourcesCacheData, @@ -27,26 +21,6 @@ import { SimpleDataSet, SimpleObject } from '../../../../../common'; * Manages caching for catalog data including data sources and accelerations. */ export class CatalogCacheManager { - /** - * Key for the data source cache in local storage. - */ - private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; - - /** - * Key for the accelerations cache in local storage. - */ - private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; - - /** - * Key for external datasources cache in local storage - */ - private static readonly externalDataSourcesCacheKey = ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE; - - /** - * Key for recently selected datasets in local storage - */ - private static readonly recentDataSetCacheKey = RECENT_DATASET_OPTIONS_CACHE; - // TODO: make this an advanced setting private static readonly maxRecentDataSet = 4; @@ -55,7 +29,7 @@ export class CatalogCacheManager { * @param {DataSourceCacheData} cacheData - The data source cache data to save. */ static saveDataSourceCache(cacheData: DataSourceCacheData): void { - sessionStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); + sessionStorage.setItem(ASYNC_QUERY.CATALOG_CACHE.KEY, JSON.stringify(cacheData)); } /** @@ -63,12 +37,12 @@ export class CatalogCacheManager { * @returns {DataSourceCacheData} The retrieved data source cache. */ static getDataSourceCache(): DataSourceCacheData { - const catalogData = sessionStorage.getItem(this.datasourceCacheKey); + const catalogData = sessionStorage.getItem(ASYNC_QUERY.CATALOG_CACHE.KEY); if (catalogData) { return JSON.parse(catalogData); } else { - const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; + const defaultCacheObject = { version: ASYNC_QUERY.CATALOG_CACHE.VERSION, dataSources: [] }; this.saveDataSourceCache(defaultCacheObject); return defaultCacheObject; } @@ -79,7 +53,7 @@ export class CatalogCacheManager { * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. */ static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { - sessionStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); + sessionStorage.setItem(ASYNC_QUERY.ACCELERATIONS_CACHE, JSON.stringify(cacheData)); } /** @@ -87,13 +61,13 @@ export class CatalogCacheManager { * @returns {AccelerationsCacheData} The retrieved accelerations cache. */ static getAccelerationsCache(): AccelerationsCacheData { - const accelerationCacheData = sessionStorage.getItem(this.accelerationsCacheKey); + const accelerationCacheData = sessionStorage.getItem(ASYNC_QUERY.ACCELERATIONS_CACHE); if (accelerationCacheData) { return JSON.parse(accelerationCacheData); } else { const defaultCacheObject = { - version: CATALOG_CACHE_VERSION, + version: ASYNC_QUERY.CATALOG_CACHE.VERSION, dataSources: [], }; this.saveAccelerationsCache(defaultCacheObject); @@ -171,19 +145,11 @@ export class CatalogCacheManager { */ static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { const cacheData = this.getDataSourceCache(); - let index; - if (dataSourceMDSId) { - index = cacheData.dataSources.findIndex( - (ds: CachedDataSource) => - ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId - ); - } - index = cacheData.dataSources.findIndex((ds: CachedDataSource) => ds.name === dataSource.name); - if (index !== -1) { - cacheData.dataSources[index] = dataSource; - } else { - cacheData.dataSources.push(dataSource); - } + const index = cacheData.dataSources.findIndex( + (ds) => + ds.name === dataSource.name && (!dataSourceMDSId || ds.dataSourceMDSId === dataSourceMDSId) + ); + cacheData.dataSources.splice(index, 1, dataSource); this.saveDataSourceCache(cacheData); } @@ -270,11 +236,11 @@ export class CatalogCacheManager { ): SimpleObject { const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); - const cachedTable = cachedDatabase.tables!.find((table) => table.title === tableName); + const cachedTable = cachedDatabase.tables!.find((table) => table.name === tableName); if (!cachedTable) { throw new Error('Table not found exception: ' + tableName); } - return cachedTable; + return { id: cachedTable.name, ...cachedTable }; } /** @@ -316,7 +282,7 @@ export class CatalogCacheManager { * Clears the data source cache from local storage. */ static clearDataSourceCache(): void { - sessionStorage.removeItem(this.datasourceCacheKey); + sessionStorage.removeItem(ASYNC_QUERY.CATALOG_CACHE.KEY); this.clearExternalDataSourcesCache(); } @@ -324,22 +290,22 @@ export class CatalogCacheManager { * Clears the accelerations cache from local storage. */ static clearAccelerationsCache(): void { - sessionStorage.removeItem(this.accelerationsCacheKey); + sessionStorage.removeItem(ASYNC_QUERY.ACCELERATIONS_CACHE); } static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { - sessionStorage.setItem(this.externalDataSourcesCacheKey, JSON.stringify(cacheData)); + sessionStorage.setItem(ASYNC_QUERY.CATALOG_CACHE.KEY, JSON.stringify(cacheData)); } static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { - const externalDataSourcesData = sessionStorage.getItem(this.externalDataSourcesCacheKey); + const externalDataSourcesData = sessionStorage.getItem(ASYNC_QUERY.CATALOG_CACHE.KEY); if (externalDataSourcesData) { return JSON.parse(externalDataSourcesData); } else { const defaultCacheObject: ExternalDataSourcesCacheData = { - version: CATALOG_CACHE_VERSION, - externalDataSources: [], + version: ASYNC_QUERY.CATALOG_CACHE.VERSION, + dataSources: [], lastUpdated: '', status: CachedDataSourceStatus.Empty, }; @@ -351,7 +317,7 @@ export class CatalogCacheManager { static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { const currentTime = new Date().toUTCString(); const cacheData = this.getExternalDataSourcesCache(); - cacheData.externalDataSources = externalDataSources; + cacheData.dataSources = externalDataSources; cacheData.lastUpdated = currentTime; cacheData.status = CachedDataSourceStatus.Updated; this.saveExternalDataSourcesCache(cacheData); @@ -362,7 +328,7 @@ export class CatalogCacheManager { } static clearExternalDataSourcesCache(): void { - sessionStorage.removeItem(this.externalDataSourcesCacheKey); + sessionStorage.removeItem(ASYNC_QUERY.CATALOG_CACHE.KEY); } static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { @@ -372,17 +338,17 @@ export class CatalogCacheManager { } static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { - sessionStorage.setItem(this.recentDataSetCacheKey, JSON.stringify(cacheData)); + sessionStorage.setItem(DATASET.OPTIONS_CACHE.KEY, JSON.stringify(cacheData)); } static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { - const recentDataSetOptionsData = sessionStorage.getItem(this.recentDataSetCacheKey); + const recentDataSetOptionsData = sessionStorage.getItem(DATASET.OPTIONS_CACHE.KEY); if (recentDataSetOptionsData) { return JSON.parse(recentDataSetOptionsData); } else { const defaultCacheObject: RecentDataSetOptionsCacheData = { - version: CATALOG_CACHE_VERSION, + version: ASYNC_QUERY.CATALOG_CACHE.VERSION, recentDataSets: [], }; this.saveRecentDataSetsCache(defaultCacheObject); @@ -411,6 +377,6 @@ export class CatalogCacheManager { } static clearRecentDataSetsCache(): void { - sessionStorage.removeItem(this.recentDataSetCacheKey); + sessionStorage.removeItem(DATASET.OPTIONS_CACHE.KEY); } } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts index e22da95ff4c6..eb970b791b3c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts @@ -3,99 +3,115 @@ * SPDX-License-Identifier: Apache-2.0 */ -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE = 'async_query_external_datasources_cache'; -export const RECENT_DATASET_OPTIONS_CACHE = 'recent_dataset_options_cache'; +export const ASYNC_QUERY = { + SEARCH_STRATEGY: 'sqlasyncraw', + SESSION_ID: { + KEY: 'async-query-session-id', + }, + CATALOG_CACHE: { + KEY: 'async-query-catalog-cache', + VERSION: '1.0', + }, + ACCELERATIONS_CACHE: 'async-query-acclerations-cache', + POLLING_INTERVAL: 2000, +}; -export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; -export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; -export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; -export const INDEX_URL_PARAM_KEY = 'indexPattern'; -export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; -export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; -export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; -export const enum QUERY_LANGUAGE { - PPL = 'PPL', - SQL = 'SQL', - DQL = 'DQL', -} -export enum DATA_SOURCE_TYPES { - DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, - SPARK = 'spark', - S3Glue = 's3glue', -} -export const ASYNC_POLLING_INTERVAL = 2000; +export const DATASET = { + OPTIONS_CACHE: { + KEY: 'recent_dataset_options_cache', + }, +}; -export const CATALOG_CACHE_VERSION = '1.0'; -export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; -export const ACCELERATION_TIME_INTERVAL = [ - { text: 'millisecond(s)', value: 'millisecond' }, - { text: 'second(s)', value: 'second' }, - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; -export const ACCELERATION_REFRESH_TIME_INTERVAL = [ - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; +export const DEFAULT_DATA_SOURCE = { + TYPE: 'DEFAULT_INDEX_PATTERNS', + NAME: 'Default cluster', + TITLE: 'Default Group', +}; -export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; -export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; -export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; -export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; -export const SANITIZE_QUERY_REGEX = /\s+/g; -export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; -export const SPARK_STRING_DATATYPE = 'string'; +export const ACCELERATION = { + DEFUALT_SKIPPING_INDEX_NAME: 'skipping', + TIME_INTERVAL: [ + { text: 'millisecond(s)', value: 'millisecond' }, + { text: 'second(s)', value: 'second' }, + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, + ], + REFRESH_TIME_INTERVAL: [ + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, + ], + ADD_FIELDS_TEXT: '(add fields here)', + INDEX_NAME_REGEX: /^[a-z0-9_]+$/, + S3_URL_REGEX: /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/, + INDEX_TYPES: [ + { label: 'Skipping Index', value: 'skipping' }, + { label: 'Covering Index', value: 'covering' }, + { label: 'Materialized View', value: 'materialized' }, + ], + INDEX_NAME_INFO: `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. + ##### Skipping Index + - For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. + ##### Covering Index + - 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. + ##### Materialized View Index + - 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. + ##### Note: + - All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `, + AGGREGRATION_FUNCTIONS: [ + { label: 'window.start' }, + { label: 'count' }, + { label: 'sum' }, + { label: 'avg' }, + { label: 'max' }, + { label: 'min' }, + ], +}; -export const ACCELERATION_INDEX_TYPES = [ - { label: 'Skipping Index', value: 'skipping' }, - { label: 'Covering Index', value: 'covering' }, - { label: 'Materialized View', value: 'materialized' }, -]; +export const SKIPPING_INDEX = { + ACCELERATION_METHODS: [ + { value: 'PARTITION', text: 'Partition' }, + { value: 'VALUE_SET', text: 'Value Set' }, + { value: 'MIN_MAX', text: 'Min Max' }, + { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, + ], +}; -export const ACC_INDEX_TYPE_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; -export const ACC_CHECKPOINT_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; +export const SPARK = { + HIVE_TABLE_REGEX: /Provider:\s*hive/, + TIMESTAMP_DATATYPE: 'timestamp', + STRING_DATATYPE: 'string', + PARTITION_INFO: `# Partition Information`, +}; -export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint___
_\`. Additionally, they may have a suffix that varies based on the index type. -##### Skipping Index -- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. - - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. -##### Covering Index -- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. - - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. -##### Materialized View Index -- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. - - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. -##### Note: -- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. - `; +export const REGEX = { + SANITIZE_QUERY: /\s+/g, +}; -export const SKIPPING_INDEX_ACCELERATION_METHODS = [ - { value: 'PARTITION', text: 'Partition' }, - { value: 'VALUE_SET', text: 'Value Set' }, - { value: 'MIN_MAX', text: 'Min Max' }, - { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, -]; +export const DOCUMENTATION = { + ACC_INDEX_TYPE_URL: + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md', + ACC_CHECKPOINT_URL: + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options', +}; -export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ - { label: 'window.start' }, - { label: 'count' }, - { label: 'sum' }, - { label: 'avg' }, - { label: 'max' }, - { label: 'min' }, -]; +export const OBSERVABILITY = { + DEFAULT_CLUSTER: 'observability-default', + S3_DATA_SOURCE: 'observability-s3', +}; -export const SPARK_PARTITION_INFO = `# Partition Information`; -export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector -export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector -export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; +export const S3_DATA_SOURCE = { + GROUP_DISPLAY_NAME: 'Amazon S3', + GROUP_SPARK_DISPLAY_NAME: 'Spark', +}; + +export const SECURITY = { + DASHBOARDS_LOGOUT_URL: '/logout', +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx index 98c2ef4e9f92..3e55b949c98d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx @@ -4,5 +4,4 @@ */ export * from './catalog_cache'; -export * from './requests'; export * from './utils'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx deleted file mode 100644 index 3918a896bd0b..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './sql'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts deleted file mode 100644 index f2c9c30c79b9..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpStart } from 'opensearch-dashboards/public'; -import { DirectQueryRequest } from '../types'; - -export class SQLService { - private http: HttpStart; - - constructor(http: HttpStart) { - this.http = http; - } - - fetch = async ( - params: DirectQueryRequest, - dataSourceMDSId?: string, - errorHandler?: (error: any) => void - ) => { - const query = { - dataSourceMDSId, - }; - return this.http - .post('/api/observability/query/jobs', { - body: JSON.stringify(params), - query, - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - fetchWithJobId = async ( - params: { queryId: string }, - dataSourceMDSId?: string, - errorHandler?: (error: any) => void - ) => { - return this.http - .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) - .catch((error) => { - // eslint-disable-next-line no-console - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - deleteWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { - return this.http.delete(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { - // eslint-disable-next-line no-console - console.error('delete error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx index 6566b2ebe4a5..62b213933f73 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx @@ -311,7 +311,7 @@ export interface ExternalDataSource { export interface ExternalDataSourcesCacheData { version: string; - externalDataSources: ExternalDataSource[]; + dataSources: ExternalDataSource[]; lastUpdated: string; status: CachedDataSourceStatus; } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts index f64089b3b1ea..37a7d8bfad49 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts @@ -4,32 +4,48 @@ */ import { HttpStart } from 'opensearch-dashboards/public'; -import { DatasourceDetails } from '../types'; +import { CachedDataSourceStatus, DatasourceDetails, ExternalDataSource } from '../types'; +import { SimpleDataSource } from '../../../../../common'; export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { try { - await http.get('/api/dataconnections'); + await http.get('/api/enhancements/datasource/external'); return true; } catch (e) { return false; } }; -export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { - const results = await Promise.all( - connectedClusters.map(async (cluster) => { - const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); - return dataSources +export const fetchExternalDataSources = async ( + http: HttpStart, + connectedClusters: SimpleDataSource[] +): Promise => { + let externalDataSources: ExternalDataSource[] = []; + + for (const cluster of connectedClusters) { + try { + const response = await http.fetch(`../../api/enhancements/datasource/external`, { + query: { + id: cluster.id, + }, + }); + + const clusterDataSources = response .filter((dataSource: DatasourceDetails) => dataSource.connector === 'S3GLUE') .map((dataSource: DatasourceDetails) => ({ name: dataSource.name, - status: dataSource.status, - dataSourceRef: cluster, + // status: dataSource.status, + dataSourceRef: cluster.id, + status: CachedDataSourceStatus.Empty, })); - }) - ); - const flattenedResults = results.flat(); + externalDataSources = externalDataSources.concat(clusterDataSources); + } catch (error) { + // Ignore error and continue with the next cluster + } + } + + const flattenedResults = externalDataSources.flat(); const uniqueResults = Array.from( flattenedResults .reduce((map, ds) => { diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts index 7dbe7ec2d4f4..79e55007654c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts @@ -8,6 +8,5 @@ export * from './fetch_data_sources'; export * from './fetch_external_data_sources'; export * from './fetch_index_patterns'; export * from './fetch_indices'; -export * from './query_session_utils'; export * from './shared'; export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts deleted file mode 100644 index fc47c8ebd020..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { ASYNC_QUERY_SESSION_ID } from '../constants'; - -function get(obj: Record, path: string, defaultValue?: T): T { - return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; -} - -export const setAsyncSessionId = (dataSource: string, sessionId: string | null) => { - if (sessionId !== null) { - sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, sessionId); - } -}; - -export const setAsyncSessionIdByObj = (dataSource: string, obj: Record) => { - const sessionId = get(obj, 'sessionId', null); - setAsyncSessionId(dataSource, sessionId); -}; - -export const getAsyncSessionId = (dataSource: string) => { - return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts index 3e4afc94e80b..8077aa851051 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts @@ -73,7 +73,7 @@ export const DSL_SETTINGS = '/indices.getFieldSettings'; export const OBSERVABILITY_BASE = '/api/observability'; export const INTEGRATIONS_BASE = '/api/integrations'; export const JOBS_BASE = '/query/jobs'; -export const DATACONNECTIONS_BASE = '/api/dataconnections'; +export const DATACONNECTIONS_BASE = '/api/directquery/dataconnections'; export const EDIT = '/edit'; export const DATACONNECTIONS_UPDATE_STATUS = '/status'; export const SECURITY_ROLES = '/api/v1/configuration/roles'; @@ -88,7 +88,6 @@ export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; export const PPL_ENDPOINT = '/_plugins/_ppl'; export const SQL_ENDPOINT = '/_plugins/_sql'; export const DSL_ENDPOINT = '/_plugins/_dsl'; -export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; export const JOB_RESULT_ENDPOINT = '/result'; @@ -319,10 +318,6 @@ export const VISUALIZATION_ERROR = { export const S3_DATA_SOURCE_TYPE = 's3glue'; -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; -export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; - export const DIRECT_DUMMY_QUERY = 'select 1'; export const DEFAULT_START_TIME = 'now-15m'; diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 9259a34fad79..400887e51d57 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,9 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; -export { - DataSetNavigator, - setAsyncSessionId, - getAsyncSessionId, - setAsyncSessionIdByObj, -} from './dataset_navigator'; +export { DataSetNavigator } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index 96c806ad0bc3..007800738952 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -137,6 +137,22 @@ export class Settings { } } + setUserQuerySessionId(dataSourceName: string, sessionId: string | null) { + if (sessionId !== null) { + sessionStorage.setItem(`async-query-session-id_${dataSourceName}`, sessionId); + } + } + + setUserQuerySessionIdByObj = (dataSourceName: string, obj: Record) => { + const sessionId = + 'sessionId'.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || null; + this.setUserQuerySessionId(dataSourceName, sessionId); + }; + + getUserQuerySessionId = (dataSourceName: string) => { + return sessionStorage.getItem(`async-query-session-id_${dataSourceName}`); + }; + toJSON(): DataSettings { return { userQueryLanguage: this.getUserQueryLanguage(), diff --git a/src/plugins/query_enhancements/common/constants.ts b/src/plugins/query_enhancements/common/constants.ts index 57316efdf5d2..8e50d82db789 100644 --- a/src/plugins/query_enhancements/common/constants.ts +++ b/src/plugins/query_enhancements/common/constants.ts @@ -25,6 +25,8 @@ export const API = { GENERATE: `${BASE_API}/assist/generate`, }, DATA_SOURCE: { + EXTERNAL: `${BASE_API}/datasource/external`, + ASYNC_JOBS: `${BASE_API}/datasource/jobs`, CONNECTIONS: `${BASE_API}/datasource/connections`, }, }; diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index cbd04f02a431..9376d4f3bd7c 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -20,8 +20,6 @@ import { ISearchOptions, SearchInterceptor, SearchInterceptorDeps, - getAsyncSessionId, - setAsyncSessionId, } from '../../../data/public'; import { API, @@ -36,6 +34,7 @@ import { QueryEnhancementsPluginStartDependencies } from '../types'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; + protected uiService!: DataPublicPluginStart['ui']; constructor(deps: SearchInterceptorDeps) { super(deps); @@ -43,6 +42,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { deps.startServices.then(([coreStart, depsStart]) => { this.queryService = (depsStart as QueryEnhancementsPluginStartDependencies).data.query; this.aggsService = (depsStart as QueryEnhancementsPluginStartDependencies).data.search.aggs; + this.uiService = (depsStart as QueryEnhancementsPluginStartDependencies).data.ui; }); } @@ -122,7 +122,9 @@ export class SQLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta.queryConfig, ...dataSourceRef, }, - sessionId: dataSourceRef ? getAsyncSessionId(dataSourceRef.dataSourceName!) : {}, + sessionId: dataSourceRef + ? this.uiService.Settings.getUserQuerySessionId(dataSourceRef.dataSourceName!) + : {}, }; const onPollingSuccess = (pollingResult: any) => { @@ -162,7 +164,10 @@ export class SQLSearchInterceptor extends SearchInterceptor { concatMap((jobResponse) => { const df = jobResponse.body; if (dataSourceRef?.dataSourceName && df?.meta?.sessionId) { - setAsyncSessionId(dataSourceRef.dataSourceName, df?.meta?.sessionId); + this.uiService.Settings.setUserQuerySessionId( + dataSourceRef.dataSourceName, + df?.meta?.sessionId + ); } const dataFramePolling = new DataFramePolling( () => fetchDataFramePolling(dfContext, df), diff --git a/src/plugins/query_enhancements/server/index.ts b/src/plugins/query_enhancements/server/index.ts index 4d72ff3eb278..0efc8b66188a 100644 --- a/src/plugins/query_enhancements/server/index.ts +++ b/src/plugins/query_enhancements/server/index.ts @@ -18,12 +18,5 @@ export function plugin(initializerContext: PluginInitializerContext) { return new QueryEnhancementsPlugin(initializerContext); } -export { - Facet, - FacetProps, - OpenSearchPPLPlugin, - OpenSearchObservabilityPlugin, - shimStats, - shimSchemaRow, -} from './utils'; +export { Facet, FacetProps, OpenSearchEnhancements, shimStats, shimSchemaRow } from './utils'; export { QueryEnhancementsPluginSetup, QueryEnhancementsPluginStart } from './types'; diff --git a/src/plugins/query_enhancements/server/plugin.ts b/src/plugins/query_enhancements/server/plugin.ts index 6c5357af70f1..789896961550 100644 --- a/src/plugins/query_enhancements/server/plugin.ts +++ b/src/plugins/query_enhancements/server/plugin.ts @@ -18,6 +18,7 @@ import { ConfigSchema } from '../common/config'; import { defineRoutes } from './routes'; import { pplSearchStrategyProvider, + pplRawSearchStrategyProvider, sqlSearchStrategyProvider, sqlAsyncSearchStrategyProvider, } from './search'; @@ -25,10 +26,8 @@ import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, QueryEnhancementsPluginStart, - QueryEnhancementsPluginStartDependencies, } from './types'; -import { OpenSearchObservabilityPlugin, OpenSearchPPLPlugin } from './utils'; -import { pplRawSearchStrategyProvider } from './search/ppl_raw_search_strategy'; +import { OpenSearchEnhancements } from './utils'; export class QueryEnhancementsPlugin implements Plugin { @@ -43,13 +42,12 @@ export class QueryEnhancementsPlugin this.logger.debug('queryEnhancements: Setup'); const router = core.http.createRouter(); // Register server side APIs - const client = core.opensearch.legacy.createClient('opensearch_observability', { - plugins: [OpenSearchPPLPlugin, OpenSearchObservabilityPlugin], + const client = core.opensearch.legacy.createClient('opensearch_enhancements', { + plugins: [OpenSearchEnhancements], }); if (dataSource) { - dataSource.registerCustomApiSchema(OpenSearchPPLPlugin); - dataSource.registerCustomApiSchema(OpenSearchObservabilityPlugin); + dataSource.registerCustomApiSchema(OpenSearchEnhancements); } const pplSearchStrategy = pplSearchStrategyProvider(this.config$, this.logger, client); @@ -84,7 +82,7 @@ export class QueryEnhancementsPlugin dataSourceEnabled: !!dataSource, })); - defineRoutes(this.logger, router, { + defineRoutes(this.logger, router, client, { ppl: pplSearchStrategy, sql: sqlSearchStrategy, sqlasync: sqlAsyncSearchStrategy, diff --git a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts index 162cc7e8f103..8407c20314bc 100644 --- a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts +++ b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts @@ -4,10 +4,13 @@ */ import { schema } from '@osd/config-schema'; -import { IRouter } from 'opensearch-dashboards/server'; +import { IRouter, ILegacyClusterClient } from 'opensearch-dashboards/server'; import { API } from '../../../common'; -export function registerDataSourceConnectionsRoutes(router: IRouter) { +export function registerDataSourceConnectionsRoutes( + router: IRouter, + defaultClient: ILegacyClusterClient +) { router.get( { path: API.DATA_SOURCE.CONNECTIONS, @@ -44,4 +47,75 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { return response.ok({ body: resp }); } ); + + router.get( + { + path: `${API.DATA_SOURCE.EXTERNAL}`, + validate: { + query: schema.object({ + id: schema.string(), + name: schema.nullable(schema.string()), + }), + }, + }, + async (context, request, response) => { + const client = request.query.id + ? context.dataSource.opensearch.legacy.getClient(request.query.id).callAPI + : defaultClient.asScoped(request).callAsCurrentUser; + + const resp = request.query.name + ? await client('enhancements.getDataConnectionById', { + dataconnection: request.query.name, + }) + : await client('enhancements.getDataConnections'); + return response.ok({ body: resp }); + } + ); + + router.get( + { + path: `${API.DATA_SOURCE.ASYNC_JOBS}`, + validate: { + query: schema.object({ + id: schema.string(), + queryId: schema.nullable(schema.string()), + }), + }, + }, + async (context, request, response) => { + const client = request.query.id + ? context.dataSource.opensearch.legacy.getClient(request.query.id).callAPI + : defaultClient.asScoped(request).callAsCurrentUser; + + const resp = await client('enhancements.getJobStatus', { + queryId: request.query.queryId, + }); + return response.ok({ body: resp }); + } + ); + + router.post( + { + path: `${API.DATA_SOURCE.ASYNC_JOBS}`, + validate: { + query: schema.object({ + id: schema.string(), + }), + body: schema.object({ + query: schema.string(), + datasource: schema.string(), + lang: schema.string(), + sessionId: schema.nullable(schema.string()), + }), + }, + }, + async (context, request, response) => { + const client = request.query.id + ? context.dataSource.opensearch.legacy.getClient(request.query.id).callAPI + : defaultClient.asScoped(request).callAsCurrentUser; + + const resp = await client('enhancements.runDirectQuery', { body: request.body }); + return response.ok({ body: resp }); + } + ); } diff --git a/src/plugins/query_enhancements/server/routes/index.ts b/src/plugins/query_enhancements/server/routes/index.ts index 8feaecc7b282..3c23db3c87b9 100644 --- a/src/plugins/query_enhancements/server/routes/index.ts +++ b/src/plugins/query_enhancements/server/routes/index.ts @@ -126,6 +126,7 @@ function defineRoute( export function defineRoutes( logger: Logger, router: IRouter, + client: any, searchStrategies: Record< string, ISearchStrategy @@ -134,6 +135,6 @@ export function defineRoutes( defineRoute(logger, router, searchStrategies, SEARCH_STRATEGY.PPL); defineRoute(logger, router, searchStrategies, SEARCH_STRATEGY.SQL); defineRoute(logger, router, searchStrategies, SEARCH_STRATEGY.SQL_ASYNC); - registerDataSourceConnectionsRoutes(router); + registerDataSourceConnectionsRoutes(router, client); registerQueryAssistRoutes(router); } diff --git a/src/plugins/query_enhancements/server/search/index.ts b/src/plugins/query_enhancements/server/search/index.ts index b3a528c57055..129ce971662f 100644 --- a/src/plugins/query_enhancements/server/search/index.ts +++ b/src/plugins/query_enhancements/server/search/index.ts @@ -4,5 +4,6 @@ */ export { pplSearchStrategyProvider } from './ppl_search_strategy'; +export { pplRawSearchStrategyProvider } from './ppl_raw_search_strategy'; export { sqlSearchStrategyProvider } from './sql_search_strategy'; export { sqlAsyncSearchStrategyProvider } from './sql_async_search_strategy'; diff --git a/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts b/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts index 72508db5d7f6..32643f621874 100644 --- a/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts @@ -21,7 +21,9 @@ export const pplRawSearchStrategyProvider = ( : client.asScoped(request.rawRequest).callAsCurrentUser; try { - const rawResponse: any = await runSearch('ppl.pplQuery', { body: request.params.body }); + const rawResponse: any = await runSearch('enhancements.pplQuery', { + body: request.params.body, + }); const data = shimSchemaRow(rawResponse); rawResponse.jsonData = data.jsonData; diff --git a/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts b/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts index 3d12448d58b0..f0b01cd51da4 100644 --- a/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts @@ -27,7 +27,7 @@ export const pplSearchStrategyProvider = ( const pplFacet = new Facet({ client, logger, - endpoint: 'ppl.pplQuery', + endpoint: 'enhancements.pplQuery', useJobs: false, shimResponse: true, }); diff --git a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts index 1a76dbf85dd9..3c3802eae261 100644 --- a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts @@ -15,6 +15,7 @@ import { createDataFrame, } from '../../../data/common'; import { Facet } from '../utils'; +import { SEARCH_STRATEGY } from '../../common'; export const sqlAsyncSearchStrategyProvider = ( config$: Observable, @@ -22,11 +23,15 @@ export const sqlAsyncSearchStrategyProvider = ( client: ILegacyClusterClient, usage?: SearchUsage ): ISearchStrategy => { - const sqlAsyncFacet = new Facet({ client, logger, endpoint: 'observability.runDirectQuery' }); + const sqlAsyncFacet = new Facet({ + client, + logger, + endpoint: 'enhancements.runDirectQuery', + }); const sqlAsyncJobsFacet = new Facet({ client, logger, - endpoint: 'observability.getJobStatus', + endpoint: 'enhancements.getJobStatus', useJobs: true, }); @@ -39,7 +44,7 @@ export const sqlAsyncSearchStrategyProvider = ( request.body = { query: request.body.query.qs, datasource: df?.meta?.queryConfig?.dataSourceName, - lang: 'sql', + lang: SEARCH_STRATEGY.SQL, sessionId: df?.meta?.sessionId, }; const rawResponse: any = await sqlAsyncFacet.describeQuery(context, request); @@ -60,6 +65,7 @@ export const sqlAsyncSearchStrategyProvider = ( }; const dataFrame = createDataFrame(partial); dataFrame.meta = { + ...dataFrame.meta, query: request.body.query, queryId, sessionId, @@ -76,7 +82,7 @@ export const sqlAsyncSearchStrategyProvider = ( const asyncResponse: any = await sqlAsyncJobsFacet.describeQuery(context, request); const status = asyncResponse.data.status; const partial: PartialDataFrame = { - name: '', + ...request.body.df, fields: asyncResponse?.data?.schema || [], }; const dataFrame = createDataFrame(partial); @@ -87,6 +93,7 @@ export const sqlAsyncSearchStrategyProvider = ( dataFrame.size = asyncResponse?.data?.datarows?.length || 0; dataFrame.meta = { + ...dataFrame.meta, status, queryId, error: status === 'FAILED' && asyncResponse.data?.error, diff --git a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts index 4566e49b0664..b2f6af9ca144 100644 --- a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts @@ -22,7 +22,7 @@ export const sqlSearchStrategyProvider = ( client: ILegacyClusterClient, usage?: SearchUsage ): ISearchStrategy => { - const sqlFacet = new Facet({ client, logger, endpoint: 'ppl.sqlQuery' }); + const sqlFacet = new Facet({ client, logger, endpoint: 'enhancements.sqlQuery' }); return { search: async (context, request: any, _options) => { diff --git a/src/plugins/query_enhancements/server/types.ts b/src/plugins/query_enhancements/server/types.ts index b6a03b672de9..5e8ab7987de9 100644 --- a/src/plugins/query_enhancements/server/types.ts +++ b/src/plugins/query_enhancements/server/types.ts @@ -5,8 +5,6 @@ import { PluginSetup } from 'src/plugins/data/server'; import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; -import { Logger } from '../../../core/server'; -import { ConfigSchema } from '../common/config'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface QueryEnhancementsPluginSetup {} @@ -18,6 +16,9 @@ export interface QueryEnhancementsPluginSetupDependencies { dataSource?: DataSourcePluginSetup; } +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface QueryEnhancementsPluginStartDependencies {} + export interface ISchema { name: string; type: string; diff --git a/src/plugins/query_enhancements/server/utils/facet.ts b/src/plugins/query_enhancements/server/utils/facet.ts index a6f23efba2a1..e930ce612d9b 100644 --- a/src/plugins/query_enhancements/server/utils/facet.ts +++ b/src/plugins/query_enhancements/server/utils/facet.ts @@ -36,14 +36,14 @@ export class Facet { endpoint: string ): Promise => { try { - const { format, df, ...query } = request.body; + const { format, df, dataSourceId, ...query } = request.body; const params = { body: { ...query }, ...(format !== 'jdbc' && { format }), }; - const dataSourceId = df?.meta?.queryConfig?.dataSourceId; - const client = dataSourceId - ? context.dataSource.opensearch.legacy.getClient(dataSourceId).callAPI + const clientId = dataSourceId ?? df?.meta?.queryConfig?.dataSourceId; + const client = clientId + ? context.dataSource.opensearch.legacy.getClient(clientId).callAPI : this.defaultClient.asScoped(request).callAsCurrentUser; const queryRes = await client(endpoint, params); return { @@ -66,10 +66,10 @@ export class Facet { ): Promise => { try { const params = request.params; - const { df } = request.body; - const dataSourceId = df?.meta?.queryConfig?.dataSourceId; - const client = dataSourceId - ? context.dataSource.opensearch.legacy.getClient(dataSourceId).callAPI + const { df, dataSourceId } = request.body; + const clientId = dataSourceId ?? df?.meta?.queryConfig?.dataSourceId; + const client = clientId + ? context.dataSource.opensearch.legacy.getClient(clientId).callAPI : this.defaultClient.asScoped(request).callAsCurrentUser; const queryRes = await client(endpoint, params); return { diff --git a/src/plugins/query_enhancements/server/utils/index.ts b/src/plugins/query_enhancements/server/utils/index.ts index 86f216973771..5cb088ed2fa3 100644 --- a/src/plugins/query_enhancements/server/utils/index.ts +++ b/src/plugins/query_enhancements/server/utils/index.ts @@ -4,6 +4,6 @@ */ export { Facet, FacetProps } from './facet'; -export { OpenSearchPPLPlugin, OpenSearchObservabilityPlugin } from './plugins'; +export { OpenSearchEnhancements } from './plugins'; export { shimStats } from './shim_stats'; export { shimSchemaRow } from './shim_schema_row'; diff --git a/src/plugins/query_enhancements/server/utils/plugins.ts b/src/plugins/query_enhancements/server/utils/plugins.ts index e8d581bc42f2..bef3bd52099b 100644 --- a/src/plugins/query_enhancements/server/utils/plugins.ts +++ b/src/plugins/query_enhancements/server/utils/plugins.ts @@ -45,51 +45,46 @@ const createAction = ( }); }; -export const OpenSearchPPLPlugin = (client: any, config: any, components: any) => { - client.prototype.ppl = components.clientAction.namespaceFactory(); - const ppl = client.prototype.ppl.prototype; +export const OpenSearchEnhancements = (client: any, config: any, components: any) => { + client.prototype.enhancements = components.clientAction.namespaceFactory(); + const enhancements = client.prototype.enhancements.prototype; - ppl.pplQuery = createAction(client, components, { + enhancements.pplQuery = createAction(client, components, { endpoint: URI.PPL, method: 'POST', needBody: true, }); - ppl.sqlQuery = createAction(client, components, { + enhancements.sqlQuery = createAction(client, components, { endpoint: URI.SQL, method: 'POST', needBody: true, }); - ppl.getDataConnectionById = createAction(client, components, { + enhancements.getDataConnectionById = createAction(client, components, { endpoint: OPENSEARCH_API.DATA_CONNECTIONS, method: 'GET', paramKey: 'dataconnection', }); - ppl.deleteDataConnection = createAction(client, components, { + enhancements.deleteDataConnection = createAction(client, components, { endpoint: OPENSEARCH_API.DATA_CONNECTIONS, method: 'DELETE', paramKey: 'dataconnection', }); - ppl.createDataSource = createAction(client, components, { + enhancements.createDataSource = createAction(client, components, { endpoint: OPENSEARCH_API.DATA_CONNECTIONS, method: 'POST', needBody: true, }); - ppl.modifyDataConnection = createAction(client, components, { + enhancements.modifyDataConnection = createAction(client, components, { endpoint: OPENSEARCH_API.DATA_CONNECTIONS, method: 'PATCH', needBody: true, }); - ppl.getDataConnections = createAction(client, components, { + enhancements.getDataConnections = createAction(client, components, { endpoint: OPENSEARCH_API.DATA_CONNECTIONS, method: 'GET', }); -}; - -export const OpenSearchObservabilityPlugin = (client: any, config: any, components: any) => { - client.prototype.observability = components.clientAction.namespaceFactory(); - const observability = client.prototype.observability.prototype; - observability.getObject = createAction(client, components, { + enhancements.getObject = createAction(client, components, { endpoint: OPENSEARCH_API.PANELS, method: 'GET', params: { @@ -106,32 +101,32 @@ export const OpenSearchObservabilityPlugin = (client: any, config: any, componen }, }); - observability.getObjectById = createAction(client, components, { + enhancements.getObjectById = createAction(client, components, { endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, method: 'GET', paramKey: 'objectId', }); - observability.createObject = createAction(client, components, { + enhancements.createObject = createAction(client, components, { endpoint: OPENSEARCH_API.PANELS, method: 'POST', needBody: true, }); - observability.updateObjectById = createAction(client, components, { + enhancements.updateObjectById = createAction(client, components, { endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, method: 'PUT', paramKey: 'objectId', needBody: true, }); - observability.deleteObjectById = createAction(client, components, { + enhancements.deleteObjectById = createAction(client, components, { endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, method: 'DELETE', paramKey: 'objectId', }); - observability.deleteObjectByIdList = createAction(client, components, { + enhancements.deleteObjectByIdList = createAction(client, components, { endpoint: OPENSEARCH_API.PANELS, method: 'DELETE', params: { @@ -139,19 +134,19 @@ export const OpenSearchObservabilityPlugin = (client: any, config: any, componen }, }); - observability.getJobStatus = createAction(client, components, { + enhancements.getJobStatus = createAction(client, components, { endpoint: `${URI.ASYNC_QUERY}`, method: 'GET', paramKey: 'queryId', }); - observability.deleteJob = createAction(client, components, { + enhancements.deleteJob = createAction(client, components, { endpoint: `${URI.ASYNC_QUERY}/<%=queryId%>`, method: 'DELETE', paramKey: 'queryId', }); - observability.runDirectQuery = createAction(client, components, { + enhancements.runDirectQuery = createAction(client, components, { endpoint: URI.ASYNC_QUERY, method: 'POST', needBody: true,