From c03c676c7159a2ceeaa797047e148819b1c66d58 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Mon, 25 Mar 2024 11:57:21 -0700 Subject: [PATCH] acceleration bug fixes Signed-off-by: Shenoy Pratik --- common/constants/data_sources.ts | 5 +- .../create_acceleration.test.tsx.snap | 71 +++++++++++- .../create/create_acceleration.tsx | 37 +++++-- .../create/create_acceleration_button.tsx | 3 +- .../index_setting_options.test.tsx.snap | 100 +++++++++++++++++ .../selectors/index_setting_options.tsx | 13 ++- .../selectors/preview_sql_defintion.tsx | 2 +- .../selectors/source_selector.tsx | 32 ++++-- .../query_visual_editor.test.tsx.snap | 8 +- .../group_by_tumble_expression.tsx | 4 +- .../skipping_index_builder.test.tsx.snap | 8 +- .../skipping_index/skipping_index_builder.tsx | 13 ++- .../framework/catalog_cache/cache_loader.tsx | 104 ++++++++++-------- 13 files changed, 303 insertions(+), 97 deletions(-) diff --git a/common/constants/data_sources.ts b/common/constants/data_sources.ts index e02243d56..ad36822b4 100644 --- a/common/constants/data_sources.ts +++ b/common/constants/data_sources.ts @@ -39,7 +39,8 @@ export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; export const SANITIZE_QUERY_REGEX = /\s+/g; -export const TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_STRING_DATATYPE = 'string'; export const ACCELERATION_INDEX_TYPES = [ { label: 'Skipping Index', value: 'skipping' }, @@ -49,6 +50,8 @@ export const ACCELERATION_INDEX_TYPES = [ export const ACC_INDEX_TYPE_DOCUMENTATION_URL = 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; +export const ACC_CHECKPOINT_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. ##### Skipping Index diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap index 5e0c5a775..d5b4b8e5c 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap @@ -655,6 +655,39 @@ Array [ > Checkpoint location + +
+ + Learn more + + + (opens in a new tab or window) + + +
- Please add fields + You have no definitions defined.
@@ -2081,6 +2114,40 @@ Array [ > Checkpoint location + +
- Please add fields + You have no definitions defined.
diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx index c6f3ddbb4..9957d16dd 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx @@ -28,6 +28,7 @@ import { DirectQueryLoadingStatus } from '../../../../../../../../common/types/e import { useLoadTableColumnsToCache } from '../../../../../../../framework/catalog_cache/cache_loader'; import { CatalogCacheManager } from '../../../../../../../framework/catalog_cache/cache_manager'; import { coreRefs } from '../../../../../../../framework/core_refs'; +import { useToast } from '../../../../../../common/toast'; import { IndexAdvancedSettings } from '../selectors/index_advanced_settings'; import { IndexSettingOptions } from '../selectors/index_setting_options'; import { IndexTypeSelector } from '../selectors/index_type_selector'; @@ -51,6 +52,7 @@ export const CreateAcceleration = ({ databaseName, tableName, }: CreateAccelerationProps) => { + const { setToast } = useToast(); const http = coreRefs!.http; const [accelerationFormData, setAccelerationFormData] = useState({ dataSource: selectedDatasource, @@ -135,13 +137,17 @@ export const CreateAcceleration = ({ stopLoadingTableFields(); if (dataTable !== '') { setTableFieldsLoading(true); - const cachedTable = CatalogCacheManager.getTable(dataSource, database, dataTable); - - if (cachedTable.columns) { - loadColumnsToAccelerationForm(cachedTable); - setTableFieldsLoading(false); - } else { - startLoading(dataSource, database, dataTable); + try { + const cachedTable = CatalogCacheManager.getTable(dataSource, database, dataTable); + if (cachedTable.columns) { + loadColumnsToAccelerationForm(cachedTable); + setTableFieldsLoading(false); + } else { + startLoading(dataSource, database, dataTable); + } + } catch (error) { + setToast('Your cache is outdated, refresh databases and tables', 'warning'); + console.error(error); } } }; @@ -159,11 +165,18 @@ export const CreateAcceleration = ({ useEffect(() => { const status = loadStatus.toLowerCase(); if (status === DirectQueryLoadingStatus.SUCCESS) { - const cachedTable = CatalogCacheManager.getTable( - accelerationFormData.dataSource, - accelerationFormData.database, - accelerationFormData.dataTable - ); + let cachedTable = {} as CachedTable; + try { + cachedTable = CatalogCacheManager.getTable( + accelerationFormData.dataSource, + accelerationFormData.database, + accelerationFormData.dataTable + ); + } catch (error) { + setToast('Your cache is outdated, refresh databases and tables', 'warning'); + console.error(error); + } + loadColumnsToAccelerationForm(cachedTable); setTableFieldsLoading(false); } else if ( diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration_button.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration_button.tsx index b1429515c..a1cbd3a09 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration_button.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration_button.tsx @@ -59,13 +59,12 @@ export const CreateAccelerationButton = ({ status === DirectQueryLoadingStatus.CANCELED ) { setIsLoading(false); - setToast('Create acceleration query failed', 'success'); } }, [directqueryLoadStatus]); return ( - Create acceleration + {isLoading ? 'Creating acceleration' : 'Create acceleration'} ); }; diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap index e38d3bce5..d4b97ea70 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap @@ -124,6 +124,40 @@ Array [ > Checkpoint location + +
+ + Learn more + + + } > - Continue in Query Workbench + Edit in Query Workbench ) : ( <> diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx index f1dac3eac..d086b0c28 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx @@ -20,12 +20,13 @@ import React, { useEffect, useState } from 'react'; import { CoreStart } from '../../../../../../../../../../src/core/public'; import { DATACONNECTIONS_BASE } from '../../../../../../../../common/constants/shared'; import { + CachedDatabase, CachedDataSourceStatus, CreateAccelerationForm, } from '../../../../../../../../common/types/data_connections'; import { CatalogCacheManager } from '../../../../../../../framework/catalog_cache/cache_manager'; import { useToast } from '../../../../../../common/toast'; -import { hasError, validateDataTable, validateDatabase } from '../create/utils'; +import { hasError, validateDatabase, validateDataTable } from '../create/utils'; import { SelectorLoadDatabases } from './selector_helpers/load_databases'; import { SelectorLoadObjects } from './selector_helpers/load_objects'; @@ -111,18 +112,25 @@ export const AccelerationDataSourceSelector = ({ const loadTables = () => { if (selectedDatabase.length > 0) { - const dbCache = CatalogCacheManager.getDatabase( - accelerationFormData.dataSource, - accelerationFormData.database - ); - if (dbCache.status === CachedDataSourceStatus.Updated && dbCache.tables.length > 0) { - const tableLabels = dbCache.tables.map((tb) => ({ label: tb.name })); - setTables(tableLabels); - } else if ( - (dbCache.status === CachedDataSourceStatus.Updated && dbCache.tables.length === 0) || - dbCache.status === CachedDataSourceStatus.Empty - ) { + let dbCache = {} as CachedDatabase; + try { + dbCache = CatalogCacheManager.getDatabase( + accelerationFormData.dataSource, + accelerationFormData.database + ); + if (dbCache.status === CachedDataSourceStatus.Updated && dbCache.tables.length > 0) { + const tableLabels = dbCache.tables.map((tb) => ({ label: tb.name })); + setTables(tableLabels); + } else if ( + (dbCache.status === CachedDataSourceStatus.Updated && dbCache.tables.length === 0) || + dbCache.status === CachedDataSourceStatus.Empty + ) { + setTables([]); + } + } catch (error) { setTables([]); + setToast('Your cache is outdated, refresh databases and tables', 'warning'); + console.error(error); } setSelectedTable([]); setAccelerationFormData({ ...accelerationFormData, dataTable: '' }); diff --git a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/__tests__/__snapshots__/query_visual_editor.test.tsx.snap b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/__tests__/__snapshots__/query_visual_editor.test.tsx.snap index 85e73b484..628a20254 100644 --- a/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/__tests__/__snapshots__/query_visual_editor.test.tsx.snap +++ b/public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/__tests__/__snapshots__/query_visual_editor.test.tsx.snap @@ -241,7 +241,7 @@ Array [ - Please add fields + You have no definitions defined.
@@ -728,12 +728,6 @@ Array [ - -