diff --git a/common/constants/shared.ts b/common/constants/shared.ts index de3124c780..bf6ada49ef 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -257,7 +257,8 @@ export const VISUALIZATION_ERROR = { export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE'; export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_CATALOG_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; export const DIRECT_DUMMY_QUERY = 'select 1'; diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index 6073401eb1..afdb04f420 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -44,21 +44,14 @@ export interface AsyncApiResponse { export type PollingCallback = (statusObj: AsyncApiResponse) => void; +export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; + +export type LoadCacheType = 'databases' | 'tables' | 'accelerations'; + export enum CachedDataSourceStatus { Updated = 'Updated', Failed = 'Failed', Empty = 'Empty', - Loading = 'Loading', -} - -export enum CachedDataSourceLoadingProgress { - LoadingScheduled = 'Loading Scheduled', - LoadingDatabases = 'Loading Databases', - LoadingTables = 'Loading Tables', - LoadingAccelerations = 'Loading Accelerations', - LoadingError = 'Loading cache ran into error', - LoadingCompleted = 'Loading Completed', - LoadingStopped = 'Loading Stopped', } export interface CachedColumn { @@ -66,36 +59,43 @@ export interface CachedColumn { dataType: string; } -export interface CachedIndex { - indexName: string; -} - export interface CachedTable { name: string; columns: CachedColumn[]; - skippingIndex?: CachedIndex; - coveringIndices: CachedIndex[]; -} - -export interface CachedMaterializedView { - name: string; } export interface CachedDatabase { name: string; - materializedViews: CachedMaterializedView[]; tables: CachedTable[]; + lastUpdated: string; // Assuming date string in UTC format + status: CachedDataSourceStatus; } export interface CachedDataSource { name: string; lastUpdated: string; // Assuming date string in UTC format status: CachedDataSourceStatus; - loadingProgress: string; databases: CachedDatabase[]; } -export interface CatalogCacheData { +export interface DataSourceCacheData { version: string; dataSources: CachedDataSource[]; } + +export interface CachedAccelerations { + flintIndexName: string; + type: AccelerationIndexType; + database: string; + table: string; + indexName: string; + autoRefresh: boolean; + status: string; +} + +export interface AccelerationsCacheData { + version: string; + accelerations: CachedAccelerations[]; + lastUpdated: string; // Assuming date string in UTC format + status: CachedDataSourceStatus; +} diff --git a/common/utils/shared.ts b/common/utils/shared.ts index 7b4012c32b..9a842c15de 100644 --- a/common/utils/shared.ts +++ b/common/utils/shared.ts @@ -16,3 +16,20 @@ export function addBackticksIfNeeded(input: string): string { return '`' + input + '`'; } } + +export function combineSchemaAndDatarows( + schema: Array<{ name: string; type: string }>, + datarows: string[][] +): object[] { + const combinedData: object[] = []; + + datarows.forEach((row) => { + const rowData: { [key: string]: string } = {}; + schema.forEach((field, index) => { + rowData[field.name] = row[index]; + }); + combinedData.push(rowData); + }); + + return combinedData; +} diff --git a/public/framework/catalog_cache/cache_intercept.ts b/public/framework/catalog_cache/cache_intercept.ts index 99ece08d4e..a44256aedd 100644 --- a/public/framework/catalog_cache/cache_intercept.ts +++ b/public/framework/catalog_cache/cache_intercept.ts @@ -17,7 +17,8 @@ export function catalogCacheInterceptError(): any { httpErrorResponse.fetchOptions.path === SECURITY_PLUGIN_ACCOUNT_API ) { // Clears all user catalog cache details - CatalogCacheManager.clear(); + CatalogCacheManager.clearDataSourceCache(); + CatalogCacheManager.clearAccelerationsCache(); } }; } diff --git a/public/framework/catalog_cache/cache_loader.test.tsx b/public/framework/catalog_cache/cache_loader.test.tsx new file mode 100644 index 0000000000..331ff87545 --- /dev/null +++ b/public/framework/catalog_cache/cache_loader.test.tsx @@ -0,0 +1,265 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CachedDataSourceStatus } from '../../../common/types/data_connections'; +import { + updateAccelerationsToCache, + updateDatabasesToCache, + updateTablesToCache, +} from './cache_loader'; +import { CatalogCacheManager } from './cache_manager'; + +// Mock CatalogCacheManager +// jest.mock('./cache_manager'); + +interface LooseObject { + [key: string]: any; +} + +// Mock localStorage +const localStorageMock = (() => { + let store = {} as LooseObject; + return { + getItem(key: string) { + return store[key] || null; + }, + setItem(key: string, value: string) { + store[key] = value.toString(); + }, + removeItem(key: string) { + delete store[key]; + }, + clear() { + store = {}; + }, + }; +})(); + +Object.defineProperty(window, 'localStorage', { value: localStorageMock }); + +// // Mock the behavior of CatalogCacheManager +// const mockAddOrUpdateDataSource = jest.fn(); +// const mockGetOrCreateDataSource = jest.fn().mockImplementation((dataSourceName: string) => ({ +// name: dataSourceName, +// databases: [], +// lastUpdated: '', // or use an actual date if needed +// status: CachedDataSourceStatus.Empty, +// })); + +// // Mock the methods used by updateDatabasesToCache +// jest.mock('./cache_manager', () => ({ +// CatalogCacheManager: { +// addOrUpdateDataSource: mockAddOrUpdateDataSource, +// getOrCreateDataSource: mockGetOrCreateDataSource, +// }, +// })); + +describe('loadCacheTests', () => { + beforeEach(() => { + jest.spyOn(window.localStorage, 'setItem'); + jest.spyOn(window.localStorage, 'getItem'); + jest.spyOn(window.localStorage, 'removeItem'); + jest.spyOn(CatalogCacheManager, 'addOrUpdateDataSource'); + jest.spyOn(CatalogCacheManager, 'updateDatabase'); + jest.spyOn(CatalogCacheManager, 'saveAccelerationsCache'); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe('updateDatabasesToCache', () => { + it('should update cache with empty databases and status failed when polling result is null', () => { + const dataSourceName = 'TestDataSource'; + const pollingResult = null; + + updateDatabasesToCache(dataSourceName, pollingResult); + + // Verify that addOrUpdateDataSource is called with the correct parameters + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalledWith({ + name: dataSourceName, + databases: [], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Failed, + }); + }); + + it('should update cache with new databases when polling result is not null', () => { + const dataSourceName = 'TestDataSource'; + const pollingResult = { + schema: [{ name: 'namespace', type: 'string' }], + datarows: [['Database1'], ['Database2']], + }; + + updateDatabasesToCache(dataSourceName, pollingResult); + + // Verify that addOrUpdateDataSource is called with the correct parameters + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalledWith({ + name: dataSourceName, + databases: [ + { name: 'Database1', tables: [], lastUpdated: '', status: CachedDataSourceStatus.Empty }, + { name: 'Database2', tables: [], lastUpdated: '', status: CachedDataSourceStatus.Empty }, + ], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Updated, + }); + }); + }); + + describe('updateTablesToCache', () => { + it('should update cache with empty tables and status failed when polling result is null', () => { + const dataSourceName = 'TestDataSource'; + const databaseName = 'TestDatabase'; + const pollingResult = null; + + CatalogCacheManager.addOrUpdateDataSource({ + databases: [ + { + name: databaseName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + tables: [], + }, + ], + name: dataSourceName, + lastUpdated: new Date().toUTCString(), + status: CachedDataSourceStatus.Updated, + }); + updateTablesToCache(dataSourceName, databaseName, pollingResult); + + // Verify that updateDatabase is called with the correct parameters + expect(CatalogCacheManager.updateDatabase).toHaveBeenCalledWith( + dataSourceName, + expect.objectContaining({ + name: databaseName, + tables: [], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Failed, + }) + ); + }); + + it('should update cache with new tables when polling result is not null', () => { + const dataSourceName = 'TestDataSource'; + const databaseName = 'TestDatabase'; + const pollingResult = { + schema: [ + { name: 'namespace', type: 'string' }, + { name: 'tableName', type: 'string' }, + { name: 'isTemporary', type: 'boolean' }, + ], + datarows: [ + ['TestDatabase', 'Table1', false], + ['TestDatabase', 'Table2', false], + ], + }; + + CatalogCacheManager.addOrUpdateDataSource({ + databases: [ + { + name: databaseName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + tables: [], + }, + ], + name: dataSourceName, + lastUpdated: new Date().toUTCString(), + status: CachedDataSourceStatus.Updated, + }); + updateTablesToCache(dataSourceName, databaseName, pollingResult); + + // Verify that updateDatabase is called with the correct parameters + expect(CatalogCacheManager.updateDatabase).toHaveBeenCalledWith( + dataSourceName, + expect.objectContaining({ + name: databaseName, + tables: [ + { name: 'Table1', columns: [] }, + { name: 'Table2', columns: [] }, + ], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Updated, + }) + ); + }); + }); + + describe('updateAccelerationsToCache', () => { + beforeEach(() => { + // Clear mock calls before each test + jest.clearAllMocks(); + }); + + it('should save empty accelerations cache and status failed when polling result is null', () => { + const pollingResult = null; + + updateAccelerationsToCache(pollingResult); + + // Verify that saveAccelerationsCache is called with the correct parameters + expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ + version: '1.0', + accelerations: [], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Failed, + }); + }); + + it('should save new accelerations cache when polling result is not null', () => { + const pollingResult = { + schema: [ + { + flint_index_name: 'Index1', + kind: 'mv', + database: 'DB1', + table: 'Table1', + index_name: 'Index1', + auto_refresh: false, + status: 'Active', + }, + { + flint_index_name: 'Index2', + kind: 'skipping', + database: 'DB2', + table: 'Table2', + index_name: 'Index2', + auto_refresh: true, + status: 'Active', + }, + ], + datarows: [], + }; + + updateAccelerationsToCache(pollingResult); + + // Verify that saveAccelerationsCache is called with the correct parameters + expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ + version: '1.0', + accelerations: [ + { + flintIndexName: 'Index1', + type: 'materialized', + database: 'DB1', + table: 'Table1', + indexName: 'Index1', + autoRefresh: false, + status: 'Active', + }, + { + flintIndexName: 'Index2', + type: 'skipping', + database: 'DB2', + table: 'Table2', + indexName: 'Index2', + autoRefresh: true, + status: 'Active', + }, + ], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Updated, + }); + }); + }); +}); diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index feb806fc09..984281b498 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -5,111 +5,222 @@ import { useEffect, useState } from 'react'; import { ASYNC_POLLING_INTERVAL } from '../../../common/constants/data_sources'; -import { CachedDataSourceLoadingProgress } from '../../../common/types/data_connections'; +import { CachedDataSourceStatus, LoadCacheType } from '../../../common/types/data_connections'; import { DirectQueryLoadingStatus, DirectQueryRequest } from '../../../common/types/explorer'; import { getAsyncSessionId, setAsyncSessionId } from '../../../common/utils/query_session_utils'; -import { addBackticksIfNeeded, get as getObjValue } from '../../../common/utils/shared'; +import { + addBackticksIfNeeded, + combineSchemaAndDatarows, + get as getObjValue, +} from '../../../common/utils/shared'; import { formatError } from '../../components/event_analytics/utils'; import { usePolling } from '../../components/hooks'; import { SQLService } from '../../services/requests/sql'; import { coreRefs } from '../core_refs'; +import { CatalogCacheManager } from './cache_manager'; -enum cacheLoadingType { - Databases = 'Load Databases', - Tables = 'Load Tables', - Accelerations = 'Load Accelerations', -} +export const updateDatabasesToCache = (dataSourceName: string, pollingResult: any) => { + const cachedDataSource = CatalogCacheManager.getOrCreateDataSource(dataSourceName); + const currentTime = new Date().toUTCString(); -const runCacheLoadQuery = ( - loadingType: cacheLoadingType, + if (!pollingResult) { + CatalogCacheManager.addOrUpdateDataSource({ + ...cachedDataSource, + databases: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newDatabases = combinedData.map((row: any) => ({ + name: row.namespace, + tables: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + })); + + CatalogCacheManager.addOrUpdateDataSource({ + ...cachedDataSource, + databases: newDatabases, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }); +}; + +export const updateTablesToCache = ( + dataSourceName: string, + databaseName: string, + pollingResult: any +) => { + const cachedDatabase = CatalogCacheManager.getDatabase(dataSourceName, databaseName); + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.updateDatabase(dataSourceName, { + ...cachedDatabase, + tables: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newTables = combinedData.map((row: any) => ({ + name: row.tableName, + columns: [], + })); + + CatalogCacheManager.updateDatabase(dataSourceName, { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }); +}; + +export const updateAccelerationsToCache = (pollingResult: any) => { + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.saveAccelerationsCache({ + version: '1.0', + accelerations: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + + const newAccelerations = combinedData.map((row: any) => ({ + flintIndexName: row.flint_index_name, + type: row.kind === 'mv' ? 'materialized' : row.kind, + database: row.database, + table: row.table, + indexName: row.index_name, + autoRefresh: row.auto_refresh, + status: row.status, + })); + + CatalogCacheManager.saveAccelerationsCache({ + version: '1.0', + accelerations: newAccelerations, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }); +}; + +export const updateToCache = ( + pollResults: any, + loadCacheType: LoadCacheType, dataSourceName: string, databaseName?: string ) => { - const [loadQueryStatus, setLoadQueryStatus] = useState<'error' | 'success' | 'loading'>( - 'loading' - ); + switch (loadCacheType) { + case 'databases': + updateDatabasesToCache(dataSourceName, pollResults); + break; + case 'tables': + updateTablesToCache(dataSourceName, databaseName!, pollResults); + break; + case 'accelerations': + updateAccelerationsToCache(pollResults); + break; + default: + break; + } +}; + +export const createLoadQuery = ( + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string +) => { + let query; + switch (loadCacheType) { + case 'databases': + query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tables': + query = `SHOW TABLES IN ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( + databaseName! + )}`; + break; + case 'accelerations': + query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; + break; + default: + query = ''; + break; + } + return query; +}; + +export const useLoadToCache = (loadCacheType: LoadCacheType) => { const sqlService = new SQLService(coreRefs.http!); + const [currentDataSourceName, setCurrentDataSourceName] = useState(''); + const [currentDatabaseName, setCurrentDatabaseName] = useState(''); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.SCHEDULED + ); const { data: pollingResult, loading: _pollingLoading, error: pollingError, startPolling, - stopPolling, + stopPolling: stopLoading, } = usePolling((params) => { return sqlService.fetchWithJobId(params); }, ASYNC_POLLING_INTERVAL); - let requestPayload = {} as DirectQueryRequest; - const sessionId = getAsyncSessionId(); - - switch (loadingType) { - case cacheLoadingType.Databases: - requestPayload = { - lang: 'sql', - query: `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`, - datasource: dataSourceName, - } as DirectQueryRequest; - break; - case cacheLoadingType.Tables: - requestPayload = { - lang: 'sql', - query: `SHOW TABLES IN ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( - databaseName! - )}`, - datasource: dataSourceName, - } as DirectQueryRequest; - break; - case cacheLoadingType.Accelerations: - requestPayload = { - lang: 'sql', - query: `SHOW FLINT INDEXES`, - datasource: dataSourceName, - } as DirectQueryRequest; - break; + const startLoading = (dataSourceName: string, databaseName?: string) => { + setCurrentDataSourceName(dataSourceName); + setCurrentDatabaseName(databaseName); - default: - setLoadQueryStatus('error'); - const formattedError = formatError( - '', - 'Recieved unknown cache query type: ' + `loadingType`, - '' - ); - coreRefs.core?.notifications.toasts.addError(formattedError, { - title: 'unknown cache query type', - }); - console.error(formattedError); - break; - } + let requestPayload: DirectQueryRequest = { + lang: 'sql', + query: createLoadQuery(loadCacheType, dataSourceName, databaseName), + datasource: dataSourceName, + }; - if (sessionId) { - requestPayload.sessionId = sessionId; - } + const sessionId = getAsyncSessionId(); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } - sqlService - .fetch(requestPayload) - .then((result) => { - setAsyncSessionId(getObjValue(result, 'sessionId', null)); - if (result.queryId) { - startPolling({ - queryId: result.queryId, + sqlService + .fetch(requestPayload) + .then((result) => { + setAsyncSessionId(getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + console.error('No query id found in response'); + setLoadStatus(DirectQueryLoadingStatus.FAILED); + updateToCache(null, loadCacheType, currentDataSourceName, currentDatabaseName); + } + }) + .catch((e) => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + updateToCache(null, loadCacheType, currentDataSourceName, currentDatabaseName); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body.message + ); + coreRefs.core?.notifications.toasts.addError(formattedError, { + title: 'Query Failed', }); - } else { - console.error('No query id found in response'); - } - }) - .catch((e) => { - // stopPollingWithStatus(DirectQueryLoadingStatus.FAILED); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - e.body.message - ); - coreRefs.core?.notifications.toasts.addError(formattedError, { - title: 'Query Failed', + console.error(e); }); - console.error(e); - }); + }; useEffect(() => { // cancel direct query @@ -118,15 +229,13 @@ const runCacheLoadQuery = ( const status = anyCaseStatus?.toLowerCase(); if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { - setLoadQueryStatus('success'); - stopPolling(); - // TODO: Stop polling, update cache + setLoadStatus(status); + stopLoading(); + updateToCache(pollingResult, loadCacheType, currentDataSourceName, currentDatabaseName); } else if (status === DirectQueryLoadingStatus.FAILED) { - setLoadQueryStatus('error'); - stopPolling(); - // TODO: Stop polling, update cache with error - - // send in a toast with error message + setLoadStatus(status); + stopLoading(); + updateToCache(null, loadCacheType, currentDataSourceName, currentDatabaseName); const formattedError = formatError( '', 'The query failed to execute and the operation could not be complete.', @@ -136,44 +245,24 @@ const runCacheLoadQuery = ( title: 'Query Failed', }); } else { - setLoadQueryStatus('loading'); + setLoadStatus(status); } }, [pollingResult, pollingError]); - return { loadQueryStatus, stopPolling }; + return { loadStatus, startLoading, stopLoading }; }; -export const CacheLoader = ({ - loadingType, - dataSourceName, - databaseName, -}: { - loadingType: cacheLoadingType; - dataSourceName: string; - databaseName?: string; -}) => { - const [loadingStatus, setloadingStatus] = useState( - CachedDataSourceLoadingProgress.LoadingScheduled - ); - const [stopCurrentPolling, setStopCurrentPolling] = useState(() => () => {}); - - const stopLoadingCache = () => { - setloadingStatus(CachedDataSourceLoadingProgress.LoadingStopped); - stopCurrentPolling(); - }; - - switch (loadingType) { - case cacheLoadingType.Databases: - break; - case cacheLoadingType.Tables: - break; - case cacheLoadingType.Accelerations: - break; +export const useLoadDatabasesToCache = () => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('databases'); + return { loadStatus, startLoading, stopLoading }; +}; - default: - // TODO: raise error toast - break; - } +export const useLoadTablesToCache = () => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables'); + return { loadStatus, startLoading, stopLoading }; +}; - return { loadingStatus, stopLoadingCache }; +export const useAccelerationsToCache = () => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('accelerations'); + return { loadStatus, startLoading, stopLoading }; }; diff --git a/public/framework/catalog_cache/cache_manager.test.tsx b/public/framework/catalog_cache/cache_manager.test.tsx new file mode 100644 index 0000000000..b7a8740193 --- /dev/null +++ b/public/framework/catalog_cache/cache_manager.test.tsx @@ -0,0 +1,398 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + ASYNC_QUERY_ACCELERATIONS_CACHE, + ASYNC_QUERY_DATASOURCE_CACHE, +} from '../../../common/constants/shared'; +import { + AccelerationsCacheData, + CachedDataSource, + CachedDataSourceStatus, + CachedDatabase, + DataSourceCacheData, +} from '../../../common/types/data_connections'; +import { CatalogCacheManager } from './cache_manager'; + +interface LooseObject { + [key: string]: any; +} + +// Mock localStorage +const localStorageMock = (() => { + let store = {} as LooseObject; + return { + getItem(key: string) { + return store[key] || null; + }, + setItem(key: string, value: string) { + store[key] = value.toString(); + }, + removeItem(key: string) { + delete store[key]; + }, + clear() { + store = {}; + }, + }; +})(); + +Object.defineProperty(window, 'localStorage', { value: localStorageMock }); + +describe('CatalogCacheManager', () => { + beforeEach(() => { + jest.spyOn(window.localStorage, 'setItem'); + jest.spyOn(window.localStorage, 'getItem'); + jest.spyOn(window.localStorage, 'removeItem'); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe('saveDataSourceCache', () => { + it('should save data source cache with correct key and data', () => { + const cacheData: DataSourceCacheData = { + version: '1.0', + dataSources: [ + { + name: 'testDataSource', + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }, + ], + }; + CatalogCacheManager.saveDataSourceCache(cacheData); + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_DATASOURCE_CACHE, + JSON.stringify(cacheData) + ); + }); + + it('should overwrite existing data source cache with new data', () => { + const initialCacheData: DataSourceCacheData = { + version: '1.0', + dataSources: [ + { + name: 'testDataSource', + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }, + ], + }; + localStorage.setItem(ASYNC_QUERY_DATASOURCE_CACHE, JSON.stringify(initialCacheData)); + + const newCacheData: DataSourceCacheData = { + version: '1.1', + dataSources: [ + { + name: 'newTestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }, + ], + }; + CatalogCacheManager.saveDataSourceCache(newCacheData); + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_DATASOURCE_CACHE, + JSON.stringify(newCacheData) + ); + }); + }); + + describe('getDataSourceCache', () => { + it('should retrieve data source cache from local storage', () => { + const cacheData: DataSourceCacheData = { + version: '1.0', + dataSources: [ + { + name: 'testDataSource', + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }, + ], + }; + localStorage.setItem(ASYNC_QUERY_DATASOURCE_CACHE, JSON.stringify(cacheData)); + expect(CatalogCacheManager.getDataSourceCache()).toEqual(cacheData); + }); + + it('should return default cache object if cache is not found', () => { + const defaultCacheObject = { version: '1.0', dataSources: [] }; + localStorage.removeItem(ASYNC_QUERY_DATASOURCE_CACHE); + expect(CatalogCacheManager.getDataSourceCache()).toEqual(defaultCacheObject); + }); + }); + + describe('saveAccelerationsCache', () => { + it('should save accelerations cache to local storage', () => { + const cacheData: AccelerationsCacheData = { + version: '1.0', + accelerations: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + CatalogCacheManager.saveAccelerationsCache(cacheData); + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_ACCELERATIONS_CACHE, + JSON.stringify(cacheData) + ); + }); + }); + + describe('getAccelerationsCache', () => { + it('should retrieve accelerations cache from local storage', () => { + const cacheData: AccelerationsCacheData = { + version: '1.0', + accelerations: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + localStorage.setItem(ASYNC_QUERY_ACCELERATIONS_CACHE, JSON.stringify(cacheData)); + expect(CatalogCacheManager.getAccelerationsCache()).toEqual(cacheData); + }); + + it('should return default cache object if cache is not found', () => { + const defaultCacheObject = { + version: '1.0', + accelerations: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + }; + localStorage.removeItem(ASYNC_QUERY_ACCELERATIONS_CACHE); + expect(CatalogCacheManager.getAccelerationsCache()).toEqual(defaultCacheObject); + }); + }); + + describe('addOrUpdateDataSource', () => { + it('should add a new data source if not exists', () => { + const dataSource: CachedDataSource = { + name: 'testDataSource', + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + const cachedData = CatalogCacheManager.getDataSourceCache(); + expect(cachedData.dataSources).toContainEqual(dataSource); + }); + + it('should update an existing data source', () => { + const dataSource: CachedDataSource = { + name: 'testDataSource', + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + const updatedDataSource: CachedDataSource = { + name: 'testDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(updatedDataSource); + const cachedData = CatalogCacheManager.getDataSourceCache(); + expect(cachedData.dataSources).toContainEqual(updatedDataSource); + }); + }); + + describe('getOrCreateDataSource', () => { + it('should retrieve existing data source if exists', () => { + const dataSourceName = 'testDataSource'; + const dataSource: CachedDataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(CatalogCacheManager.getOrCreateDataSource(dataSourceName)).toEqual(dataSource); + }); + + it('should create a new data source if not exists', () => { + const dataSourceName = 'testDataSource'; + const createdDataSource: CachedDataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + expect(CatalogCacheManager.getOrCreateDataSource(dataSourceName)).toEqual(createdDataSource); + }); + }); + + describe('getDatabase', () => { + it('should retrieve database from cache', () => { + const dataSourceName = 'testDataSource'; + const databaseName = 'testDatabase'; + const database: CachedDatabase = { + name: databaseName, + tables: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + const dataSource: CachedDataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [database], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(CatalogCacheManager.getDatabase(dataSourceName, databaseName)).toEqual(database); + }); + + it('should throw error if data source not found', () => { + const dataSourceName = 'nonExistingDataSource'; + const databaseName = 'testDatabase'; + expect(() => CatalogCacheManager.getDatabase(dataSourceName, databaseName)).toThrowError( + 'DataSource not found exception: ' + dataSourceName + ); + }); + + it('should throw error if database not found', () => { + const dataSourceName = 'testDataSource'; + const databaseName = 'nonExistingDatabase'; + const dataSource: CachedDataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(() => CatalogCacheManager.getDatabase(dataSourceName, databaseName)).toThrowError( + 'Database not found exception: ' + databaseName + ); + }); + }); + + describe('getTable', () => { + it('should retrieve table from cache', () => { + const dataSourceName = 'testDataSource'; + const databaseName = 'testDatabase'; + const tableName = 'testTable'; + const table = { + name: tableName, + columns: [], + }; + const database = { + name: databaseName, + tables: [table], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + const dataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [database], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(CatalogCacheManager.getTable(dataSourceName, databaseName, tableName)).toEqual(table); + }); + + it('should throw error if table not found', () => { + const dataSourceName = 'testDataSource'; + const databaseName = 'testDatabase'; + const tableName = 'nonExistingTable'; + const dataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [ + { + name: databaseName, + tables: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Updated, + }, + ], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(() => + CatalogCacheManager.getTable(dataSourceName, databaseName, tableName) + ).toThrowError('Table not found exception: ' + tableName); + }); + }); + + describe('updateDatabase', () => { + it('should update database in cache', () => { + const dataSourceName = 'testDataSource'; + const databaseName = 'testDatabase'; + const database = { + name: databaseName, + tables: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + const updatedDatabase = { + name: databaseName, + tables: [], + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + }; + const dataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [database], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + CatalogCacheManager.updateDatabase(dataSourceName, updatedDatabase); + const cachedData = CatalogCacheManager.getDataSourceCache(); + expect(cachedData.dataSources[0].databases[0]).toEqual(updatedDatabase); + }); + + it('should throw error if data source not found', () => { + const dataSourceName = 'nonExistingDataSource'; + const database = { + name: 'testDatabase', + tables: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + expect(() => CatalogCacheManager.updateDatabase(dataSourceName, database)).toThrowError( + 'DataSource not found exception: ' + dataSourceName + ); + }); + + it('should throw error if database not found', () => { + const dataSourceName = 'testDataSource'; + const database = { + name: 'nonExistingDatabase', + tables: [], + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + }; + const dataSource = { + name: dataSourceName, + lastUpdated: '2024-03-07T12:00:00Z', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + CatalogCacheManager.addOrUpdateDataSource(dataSource); + expect(() => CatalogCacheManager.updateDatabase(dataSourceName, database)).toThrowError( + 'Database not found exception: ' + database.name + ); + }); + }); + + describe('clearDataSourceCache', () => { + it('should clear data source cache from local storage', () => { + CatalogCacheManager.clearDataSourceCache(); + expect(localStorage.removeItem).toHaveBeenCalledWith(ASYNC_QUERY_DATASOURCE_CACHE); + }); + }); + + describe('clearAccelerationsCache', () => { + it('should clear accelerations cache from local storage', () => { + CatalogCacheManager.clearAccelerationsCache(); + expect(localStorage.removeItem).toHaveBeenCalledWith(ASYNC_QUERY_ACCELERATIONS_CACHE); + }); + }); +}); diff --git a/public/framework/catalog_cache/cache_manager.ts b/public/framework/catalog_cache/cache_manager.ts index fde9193f5f..f1a0ebc053 100644 --- a/public/framework/catalog_cache/cache_manager.ts +++ b/public/framework/catalog_cache/cache_manager.ts @@ -3,166 +3,120 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { ASYNC_QUERY_CATALOG_CACHE } from '../../../common/constants/shared'; import { + ASYNC_QUERY_ACCELERATIONS_CACHE, + ASYNC_QUERY_DATASOURCE_CACHE, +} from '../../../common/constants/shared'; +import { + AccelerationsCacheData, CachedDataSource, - CachedDataSourceLoadingProgress, CachedDataSourceStatus, - CatalogCacheData, + CachedDatabase, + CachedTable, + DataSourceCacheData, } from '../../../common/types/data_connections'; /** - * - * Manages caching of catalog data in the browser storage - * - * * * * * * * * * * Example usage for CatalogCacheManager * * * * * * * * * * - * - * const dataSource: CachedDataSource = { - * name: 'DataSource1', - * lastUpdated: '2024-02-20T12:00:00Z', - * status: CachedDataSourceStatus.Empty, - * databases: [ - * { - * name: 'Database1', - * materializedViews: [{ name: 'MaterializedView1' }, { name: 'MaterializedView2' }], - * tables: [ - * { - * name: 'Table1', - * columns: [ - * { name: 'column1', dataType: 'datatype1' }, - * { name: 'column2', dataType: 'datatype2' }, - * { name: 'column3', dataType: 'datatype3' }, - * ], - * skippingIndex: { indexName: 'SkippingIndex1' }, - * coveringIndices: [{ indexName: 'CoveringIndex1' }, { indexName: 'CoveringIndex2' }], - * }, - * { - * name: 'Table2', - * columns: [ - * { name: 'column4', dataType: 'datatype4' }, - * { name: 'column5', dataType: 'datatype5' }, - * ], - * skippingIndex: { indexName: 'SkippingIndex2' }, - * coveringIndices: [], - * }, - * ], - * }, - * ], - * }; - * - * // Save the dataSource into cache - * CatalogCacheManager.addOrUpdateDataSource(dataSource); - * - * // Retrieve dataSource from cache - * const cachedDataSource = CatalogCacheManager.getDataSource('DataSource1'); - * - * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Manages caching for catalog data including data sources and accelerations. */ - export class CatalogCacheManager { /** - * The key used to store catalog cache data in localStorage. + * Key for the data source cache in local storage. */ - private static readonly localStorageKey = ASYNC_QUERY_CATALOG_CACHE; + private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; /** - * Retrieves catalog cache data from localStorage. - * If no data is found, initializes with a default cache object. - * @returns The catalog cache data. + * Key for the accelerations cache in local storage. */ - private static getCatalogCacheData(): CatalogCacheData { - const catalogData = localStorage.getItem(this.localStorageKey); + private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; + + /** + * Saves data source cache to local storage. + * @param {DataSourceCacheData} cacheData - The data source cache data to save. + */ + static saveDataSourceCache(cacheData: DataSourceCacheData): void { + localStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves data source cache from local storage. + * @returns {DataSourceCacheData} The retrieved data source cache. + */ + static getDataSourceCache(): DataSourceCacheData { + const catalogData = localStorage.getItem(this.datasourceCacheKey); if (catalogData) { return JSON.parse(catalogData); } else { const defaultCacheObject = { version: '1.0', dataSources: [] }; - this.saveCatalogCacheData(defaultCacheObject); + this.saveDataSourceCache(defaultCacheObject); return defaultCacheObject; } } /** - * Saves catalog cache data to localStorage. - * @param cacheData The catalog cache data to save. + * Saves accelerations cache to local storage. + * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. */ - static saveCatalogCacheData(cacheData: CatalogCacheData): void { - localStorage.setItem(this.localStorageKey, JSON.stringify(cacheData)); + static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { + localStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); } /** - * Loads/Refreshes a datasource in the catalog cache. - * @param dataSource The data source name to be loaded/refreshed + * Retrieves accelerations cache from local storage. + * @returns {AccelerationsCacheData} The retrieved accelerations cache. */ - static loadDataSource(dataSourceName: string): boolean { - try { - // let ds = this.getDataSource(dataSourceName); - // // TODO: Initiate loader for datasource - // ds.status = CachedDataSourceStatus.Loading; - // ds.loadingProgress = CachedDataSourceLoadingProgress.LoadingDatabases; - // this.addOrUpdateDataSource(ds); - return true; - } catch (err) { - console.error(err); - return false; - } - } + static getAccelerationsCache(): AccelerationsCacheData { + const accelerationCacheData = localStorage.getItem(this.accelerationsCacheKey); - /** - * Loads/Refreshes a tables of a database in the catalog cache. - * @param dataSource The data source name to be loaded/refreshed - * @param databaseName The database name to be loaded/refreshed - */ - static loadTables(dataSourceName: string, databaseName: string): boolean { - try { - let ds = this.getDataSource(dataSourceName); - // TODO: Initiate loader for Table - ds.status = CachedDataSourceStatus.Loading; - ds.loadingProgress = CachedDataSourceLoadingProgress.LoadingDatabases; - this.addOrUpdateDataSource(ds); - return true; - } catch (err) { - console.error(err); - return false; + if (accelerationCacheData) { + return JSON.parse(accelerationCacheData); + } else { + const defaultCacheObject = { + version: '1.0', + accelerations: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + }; + this.saveAccelerationsCache(defaultCacheObject); + return defaultCacheObject; } } /** - * Adds or updates a data source in the catalog cache. - * @param dataSource The data source to be added or updated. + * Adds or updates a data source in the cache. + * @param {CachedDataSource} dataSource - The data source to add or update. */ static addOrUpdateDataSource(dataSource: CachedDataSource): void { - const cacheData = this.getCatalogCacheData(); - const index = cacheData.dataSources.findIndex((ds) => ds.name === dataSource.name); + const cacheData = this.getDataSourceCache(); + const index = cacheData.dataSources.findIndex( + (ds: CachedDataSource) => ds.name === dataSource.name + ); if (index !== -1) { cacheData.dataSources[index] = dataSource; } else { cacheData.dataSources.push(dataSource); } - this.saveCatalogCacheData(cacheData); + this.saveDataSourceCache(cacheData); } /** - * Retrieves a data source from the catalog cache. - * If the data source does not exist, creates and returns a default data source object. - * @param dataSourceName The name of the data source to retrieve. - * @returns The cached data source, or a default data source object if not found. + * Retrieves or creates a data source with the specified name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedDataSource} The retrieved or created data source. */ - static getDataSource(dataSourceName: string): CachedDataSource { - const cacheData = this.getCatalogCacheData(); - const cachedDataSourceData = cacheData.dataSources.find( + static getOrCreateDataSource(dataSourceName: string): CachedDataSource { + const cacheData = this.getDataSourceCache(); + const cachedDataSource = cacheData.dataSources.find( (ds: CachedDataSource) => ds.name === dataSourceName ); - - if (cachedDataSourceData) { - return cachedDataSourceData; + if (cachedDataSource) { + return cachedDataSource; } else { const defaultDataSourceObject = { name: dataSourceName, lastUpdated: '', status: CachedDataSourceStatus.Empty, - loadingProgress: '', databases: [], }; this.addOrUpdateDataSource(defaultDataSourceObject); @@ -171,9 +125,80 @@ export class CatalogCacheManager { } /** - * Clears the catalog cache by removing the cache data from localStorage. + * Retrieves a database from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @returns {CachedDatabase} The retrieved database. + * @throws {Error} If the data source or database is not found. + */ + static getDatabase(dataSourceName: string, databaseName: string): CachedDatabase { + const cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); + if (!cachedDatabase) { + throw new Error('Database not found exception: ' + databaseName); + } + + return cachedDatabase; + } + + /** + * Retrieves a table from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @param {string} tableName - The name of the database. + * @returns {Cachedtable} The retrieved database. + * @throws {Error} If the data source, database or table is not found. + */ + static getTable(dataSourceName: string, databaseName: string, tableName: string): CachedTable { + const cachedDatabase = this.getDatabase(dataSourceName, databaseName); + + const cachedTable = cachedDatabase.tables.find((table) => table.name === tableName); + if (!cachedTable) { + throw new Error('Table not found exception: ' + tableName); + } + return cachedTable; + } + + /** + * Updates a database in the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {CachedDatabase} database - The database to be updated. + * @throws {Error} If the data source or database is not found. + */ + static updateDatabase(dataSourceName: string, database: CachedDatabase): void { + const cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); + if (index !== -1) { + cachedDataSource.databases[index] = database; + this.addOrUpdateDataSource(cachedDataSource); + } else { + throw new Error('Database not found exception: ' + database.name); + } + } + + /** + * Clears the data source cache from local storage. + */ + static clearDataSourceCache(): void { + localStorage.removeItem(this.datasourceCacheKey); + } + + /** + * Clears the accelerations cache from local storage. */ - static clear(): void { - localStorage.removeItem(this.localStorageKey); + static clearAccelerationsCache(): void { + localStorage.removeItem(this.accelerationsCacheKey); } }