diff --git a/common/constants/data_sources.ts b/common/constants/data_sources.ts index 791851662..7bfc5eee4 100644 --- a/common/constants/data_sources.ts +++ b/common/constants/data_sources.ts @@ -22,3 +22,5 @@ export enum DATA_SOURCE_TYPES { S3Glue = 's3glue', } export const ASYNC_POLLING_INTERVAL = 2000; + +export const CATALOG_CACHE_VERSION = '1.0'; diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index 3927131cc..ba13c2eb7 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -99,3 +99,10 @@ export interface AccelerationsCacheData { lastUpdated: string; // Assuming date string in UTC format status: CachedDataSourceStatus; } + +export interface PollingSuccessResult { + schema: Array<{ name: string; type: string }>; + datarows: Array>; +} + +export type AsyncPollingResult = PollingSuccessResult | null; diff --git a/common/utils/shared.ts b/common/utils/shared.ts index 9a842c15d..7615723d3 100644 --- a/common/utils/shared.ts +++ b/common/utils/shared.ts @@ -19,12 +19,12 @@ export function addBackticksIfNeeded(input: string): string { export function combineSchemaAndDatarows( schema: Array<{ name: string; type: string }>, - datarows: string[][] + datarows: Array> ): object[] { const combinedData: object[] = []; datarows.forEach((row) => { - const rowData: { [key: string]: string } = {}; + const rowData: { [key: string]: string | number | boolean } = {}; schema.forEach((field, index) => { rowData[field.name] = row[index]; }); diff --git a/public/framework/catalog_cache/cache_loader.test.tsx b/public/framework/catalog_cache/cache_loader.test.tsx index dfdda4d71..063da0108 100644 --- a/public/framework/catalog_cache/cache_loader.test.tsx +++ b/public/framework/catalog_cache/cache_loader.test.tsx @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { CATALOG_CACHE_VERSION } from '../../../common/constants/data_sources'; import { CachedDataSourceStatus } from '../../../common/types/data_connections'; import { mockShowDatabasesPollingResult, @@ -169,7 +170,7 @@ describe('loadCacheTests', () => { // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: expect.any(String), status: CachedDataSourceStatus.Failed, @@ -181,7 +182,7 @@ describe('loadCacheTests', () => { // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [ { flintIndexName: 'flint_mys3_default_http_logs_skipping_index', diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index 9f8f4821a..60fc1175e 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -4,8 +4,15 @@ */ import { useEffect, useState } from 'react'; -import { ASYNC_POLLING_INTERVAL } from '../../../common/constants/data_sources'; -import { CachedDataSourceStatus, LoadCacheType } from '../../../common/types/data_connections'; +import { + ASYNC_POLLING_INTERVAL, + CATALOG_CACHE_VERSION, +} from '../../../common/constants/data_sources'; +import { + AsyncPollingResult, + CachedDataSourceStatus, + LoadCacheType, +} from '../../../common/types/data_connections'; import { DirectQueryLoadingStatus, DirectQueryRequest } from '../../../common/types/explorer'; import { getAsyncSessionId, setAsyncSessionId } from '../../../common/utils/query_session_utils'; import { @@ -19,7 +26,10 @@ import { SQLService } from '../../services/requests/sql'; import { coreRefs } from '../core_refs'; import { CatalogCacheManager } from './cache_manager'; -export const updateDatabasesToCache = (dataSourceName: string, pollingResult: any) => { +export const updateDatabasesToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult +) => { const cachedDataSource = CatalogCacheManager.getOrCreateDataSource(dataSourceName); const currentTime = new Date().toUTCString(); @@ -52,7 +62,7 @@ export const updateDatabasesToCache = (dataSourceName: string, pollingResult: an export const updateTablesToCache = ( dataSourceName: string, databaseName: string, - pollingResult: any + pollingResult: AsyncPollingResult ) => { const cachedDatabase = CatalogCacheManager.getDatabase(dataSourceName, databaseName); const currentTime = new Date().toUTCString(); @@ -81,12 +91,12 @@ export const updateTablesToCache = ( }); }; -export const updateAccelerationsToCache = (pollingResult: any) => { +export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => { const currentTime = new Date().toUTCString(); if (!pollingResult) { CatalogCacheManager.saveAccelerationsCache({ - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: currentTime, status: CachedDataSourceStatus.Failed, @@ -107,7 +117,7 @@ export const updateAccelerationsToCache = (pollingResult: any) => { })); CatalogCacheManager.saveAccelerationsCache({ - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: newAccelerations, lastUpdated: currentTime, status: CachedDataSourceStatus.Updated, diff --git a/public/framework/catalog_cache/cache_manager.test.tsx b/public/framework/catalog_cache/cache_manager.test.tsx index b7a874019..69be8be29 100644 --- a/public/framework/catalog_cache/cache_manager.test.tsx +++ b/public/framework/catalog_cache/cache_manager.test.tsx @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { CATALOG_CACHE_VERSION } from '../../../common/constants/data_sources'; import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE, @@ -55,7 +56,7 @@ describe('CatalogCacheManager', () => { describe('saveDataSourceCache', () => { it('should save data source cache with correct key and data', () => { const cacheData: DataSourceCacheData = { - version: '1.0', + version: CATALOG_CACHE_VERSION, dataSources: [ { name: 'testDataSource', @@ -74,7 +75,7 @@ describe('CatalogCacheManager', () => { it('should overwrite existing data source cache with new data', () => { const initialCacheData: DataSourceCacheData = { - version: '1.0', + version: CATALOG_CACHE_VERSION, dataSources: [ { name: 'testDataSource', @@ -108,7 +109,7 @@ describe('CatalogCacheManager', () => { describe('getDataSourceCache', () => { it('should retrieve data source cache from local storage', () => { const cacheData: DataSourceCacheData = { - version: '1.0', + version: CATALOG_CACHE_VERSION, dataSources: [ { name: 'testDataSource', @@ -123,7 +124,7 @@ describe('CatalogCacheManager', () => { }); it('should return default cache object if cache is not found', () => { - const defaultCacheObject = { version: '1.0', dataSources: [] }; + const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; localStorage.removeItem(ASYNC_QUERY_DATASOURCE_CACHE); expect(CatalogCacheManager.getDataSourceCache()).toEqual(defaultCacheObject); }); @@ -132,7 +133,7 @@ describe('CatalogCacheManager', () => { describe('saveAccelerationsCache', () => { it('should save accelerations cache to local storage', () => { const cacheData: AccelerationsCacheData = { - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: '2024-03-07T12:00:00Z', status: CachedDataSourceStatus.Empty, @@ -148,7 +149,7 @@ describe('CatalogCacheManager', () => { describe('getAccelerationsCache', () => { it('should retrieve accelerations cache from local storage', () => { const cacheData: AccelerationsCacheData = { - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: '2024-03-07T12:00:00Z', status: CachedDataSourceStatus.Empty, @@ -159,7 +160,7 @@ describe('CatalogCacheManager', () => { it('should return default cache object if cache is not found', () => { const defaultCacheObject = { - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: '', status: CachedDataSourceStatus.Empty, diff --git a/public/framework/catalog_cache/cache_manager.ts b/public/framework/catalog_cache/cache_manager.ts index f1a0ebc05..b183170b0 100644 --- a/public/framework/catalog_cache/cache_manager.ts +++ b/public/framework/catalog_cache/cache_manager.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { CATALOG_CACHE_VERSION } from '../../../common/constants/data_sources'; import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE, @@ -48,7 +49,7 @@ export class CatalogCacheManager { if (catalogData) { return JSON.parse(catalogData); } else { - const defaultCacheObject = { version: '1.0', dataSources: [] }; + const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; this.saveDataSourceCache(defaultCacheObject); return defaultCacheObject; } @@ -73,7 +74,7 @@ export class CatalogCacheManager { return JSON.parse(accelerationCacheData); } else { const defaultCacheObject = { - version: '1.0', + version: CATALOG_CACHE_VERSION, accelerations: [], lastUpdated: '', status: CachedDataSourceStatus.Empty, diff --git a/test/datasources.ts b/test/datasources.ts index ac134af4b..69b2d2c55 100644 --- a/test/datasources.ts +++ b/test/datasources.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { DatasourceType } from '../common/types/data_connections'; +import { AsyncPollingResult, DatasourceType } from '../common/types/data_connections'; export const showDataConnectionsData = { schema: [ @@ -807,12 +807,12 @@ export const mockRoleData = { }, }; -export const mockShowDatabasesPollingResult = { +export const mockShowDatabasesPollingResult: AsyncPollingResult = { schema: [{ name: 'namespace', type: 'string' }], datarows: [['Database1'], ['Database2']], }; -export const mockShowTablesPollingResult = { +export const mockShowTablesPollingResult: AsyncPollingResult = { schema: [ { name: 'namespace', type: 'string' }, { name: 'tableName', type: 'string' }, @@ -824,7 +824,7 @@ export const mockShowTablesPollingResult = { ], }; -export const mockShowIndexesPollingResult = { +export const mockShowIndexesPollingResult: AsyncPollingResult = { schema: [ { name: 'flint_index_name', type: 'string' }, { name: 'kind', type: 'string' },