feat: Add support for multi-partition parquet:kv tables (#1580)

Added feature to `IrisGridPartitionSelector` to display additional partition selector dropdowns when more than one partition column is present. The values displayed in the dropdown depend on the selected partitions of columns to the left of it. The "Append Command" button and associated functions are removed in preparation for #1143. Closes #1438 #### Testing instructions: 1. Create and open a multi-partition table: ```python from deephaven import empty_table part = empty_table(4).update("II=ii") from deephaven.parquet import write, read write(part, "/tmp/pt-1438/x=0/y=0/part.parquet") write(part, "/tmp/pt-1438/x=0/y=1/part.parquet") write(part, "/tmp/pt-1438/x=1/y=0/part.parquet") write(part, "/tmp/pt-1438/x=1/y=1/part.parquet") partition_table = read("/tmp/pt-1438") ``` 2. Use the dropdown to select any combination of partitions and check that it updates the table correctly. * The dropdown should only display values that are available after filtering for all the partitions left of it. (E.g. A state/province dropdown should only contain the states/provinces corresponding to the selected country.) 3. Test filtering by typing directly in the dropdown and searching by using the search bar above the list of available partitions. 4. Retest using other data types for partition columns (dates, strings, chars, doubles). E.g: ```python from deephaven.parquet import write, read write(part, "/tmp/pt2-1438/date_partition=2023-08-01/id_partition=id-001/decimal=0.1/part.parquet") write(part, "/tmp/pt2-1438/date_partition=2023-08-01/id_partition=id-002/decimal=0.15/part.parquet") write(part, "/tmp/pt2-1438/date_partition=2023-08-02/id_partition=id-001/decimal=0.08/part.parquet") write(part, "/tmp/pt2-1438/date_partition=2023-08-02/id_partition=id-002/decimal=0.08/part.parquet") write(part, "/tmp/pt2-1438/date_partition=2023-08-02/id_partition=id-003/decimal=0.1/part.parquet") partition_table = read("/tmp/pt2-1438") ``` --------- Co-authored-by: georgecwan <georgecwan@users.noreply.github.com>
deephaven · Nov 2, 2023 · d92c91e · d92c91e
1 parent fc543aa
commit d92c91e
Show file tree

Hide file tree

Showing 13 changed files with 593 additions and 314 deletions.
diff --git a/packages/code-studio/src/styleguide/MockIrisGridTreeModel.ts b/packages/code-studio/src/styleguide/MockIrisGridTreeModel.ts
@@ -324,7 +324,7 @@ class MockIrisGridTreeModel
     throw new Error('Not defined in mock');
   }
 
-  valuesTable(column: Column): Promise<never> {
+  valuesTable(columns: Column | Column[]): Promise<never> {
     throw new Error('Not defined in mock');
   }
 

diff --git a/packages/dashboard-core-plugins/src/panels/ChartPanel.tsx b/packages/dashboard-core-plugins/src/panels/ChartPanel.tsx
@@ -113,8 +113,8 @@ export interface GLChartPanelState {
     sorts: unknown;
   };
   irisGridPanelState?: {
-    partitionColumn: string;
-    partition: unknown;
+    partitionColumns: string[];
+    partitions: unknown[];
   };
   table?: string;
   figure?: string;

diff --git a/packages/dashboard-core-plugins/src/panels/IrisGridPanel.tsx b/packages/dashboard-core-plugins/src/panels/IrisGridPanel.tsx
@@ -39,6 +39,7 @@ import {
   IrisGridState,
   ChartBuilderSettings,
   DehydratedIrisGridState,
+  DehydratedIrisGridPanelState,
   ColumnHeaderGroup,
   IrisGridContextMenuData,
   IrisGridTableModel,
@@ -78,7 +79,7 @@ import type {
   TablePluginComponent,
   TablePluginElement,
 } from '@deephaven/plugin';
-import { ConsoleEvent, InputFilterEvent, IrisGridEvent } from '../events';
+import { InputFilterEvent, IrisGridEvent } from '../events';
 import {
   getInputFiltersForDashboard,
   getLinksForDashboard,
@@ -114,23 +115,20 @@ export interface PanelState {
     movedRows: MoveOperation[];
   };
   irisGridState: DehydratedIrisGridState;
-  irisGridPanelState: {
-    partitionColumn: ColumnName | null;
-    partition: string | null;
-    isSelectingPartition: boolean;
-    advancedSettings: [AdvancedSettingsType, boolean][];
-  };
+  irisGridPanelState: DehydratedIrisGridPanelState;
   pluginState: unknown;
 }
 
 // Some of the properties in the loaded panel state may be omitted
 // even though they can't be undefined in the dehydrated state.
 // This can happen when loading the state saved before the properties were added.
 type LoadedPanelState = PanelState & {
-  irisGridPanelState: PanelState['irisGridPanelState'] &
-    Partial<
-      Pick<PanelState['irisGridPanelState'], 'partition' | 'partitionColumn'>
-    >;
+  irisGridPanelState: PanelState['irisGridPanelState'] & {
+    partitions?: (string | null)[];
+    partitionColumns?: ColumnName[];
+    partition?: string | null;
+    partitionColumn?: ColumnName | null;
+  };
 };
 
 export interface OwnProps extends DashboardPanelProps {
@@ -190,8 +188,8 @@ interface IrisGridPanelState {
   movedColumns: readonly MoveOperation[];
   movedRows: readonly MoveOperation[];
   isSelectingPartition: boolean;
-  partition: string | null;
-  partitionColumn: Column | null;
+  partitions: (string | null)[];
+  partitionColumns: Column[];
   rollupConfig?: UIRollupConfig;
   showSearchBar: boolean;
   searchValue: string;
@@ -252,7 +250,6 @@ export class IrisGridPanel extends PureComponent<
     this.handleError = this.handleError.bind(this);
     this.handleGridStateChange = this.handleGridStateChange.bind(this);
     this.handlePluginStateChange = this.handlePluginStateChange.bind(this);
-    this.handlePartitionAppend = this.handlePartitionAppend.bind(this);
     this.handleCreateChart = this.handleCreateChart.bind(this);
     this.handleResize = this.handleResize.bind(this);
     this.handleShow = this.handleShow.bind(this);
@@ -298,8 +295,8 @@ export class IrisGridPanel extends PureComponent<
       movedColumns: [],
       movedRows: [],
       isSelectingPartition: false,
-      partition: null,
-      partitionColumn: null,
+      partitions: [],
+      partitionColumns: [],
       rollupConfig: undefined,
       showSearchBar: false,
       searchValue: '',
@@ -460,14 +457,14 @@ export class IrisGridPanel extends PureComponent<
     (
       model: IrisGridModel,
       isSelectingPartition: boolean,
-      partition: string | null,
-      partitionColumn: Column | null,
+      partitions: (string | null)[],
+      partitionColumns: Column[],
       advancedSettings: Map<AdvancedSettingsType, boolean>
     ) =>
       IrisGridUtils.dehydrateIrisGridPanelState(model, {
         isSelectingPartition,
-        partition,
-        partitionColumn,
+        partitions,
+        partitionColumns,
         advancedSettings,
       })
   );
@@ -713,14 +710,6 @@ export class IrisGridPanel extends PureComponent<
     glEventHub.emit(InputFilterEvent.TABLE_CHANGED, this, table);
   }
 
-  handlePartitionAppend(column: Column, value: unknown): void {
-    const { glEventHub } = this.props;
-    const { name } = column;
-    const tableName = this.getTableName();
-    const command = `${tableName} = ${tableName}.where("${name}=\`${value}\`")`;
-    glEventHub.emit(ConsoleEvent.SEND_COMMAND, command, false, true);
-  }
-
   /**
    * Create a chart with the specified settings
    * @param settings The settings from the chart builder
@@ -1044,8 +1033,8 @@ export class IrisGridPanel extends PureComponent<
       }
       const {
         isSelectingPartition,
-        partition,
-        partitionColumn,
+        partitions,
+        partitionColumns,
         advancedSettings,
       } = IrisGridUtils.hydrateIrisGridPanelState(model, irisGridPanelState);
       assertNotNull(this.irisGridUtils);
@@ -1090,8 +1079,8 @@ export class IrisGridPanel extends PureComponent<
         isSelectingPartition,
         movedColumns,
         movedRows,
-        partition,
-        partitionColumn,
+        partitions,
+        partitionColumns,
         quickFilters,
         reverseType,
         rollupConfig,
@@ -1123,8 +1112,8 @@ export class IrisGridPanel extends PureComponent<
       model,
       panelState: oldPanelState,
       isSelectingPartition,
-      partition,
-      partitionColumn,
+      partitions,
+      partitionColumns,
       advancedSettings,
     } = this.state;
     const {
@@ -1159,8 +1148,8 @@ export class IrisGridPanel extends PureComponent<
       this.getDehydratedIrisGridPanelState(
         model,
         isSelectingPartition,
-        partition,
-        partitionColumn,
+        partitions,
+        partitionColumns,
         advancedSettings
       ),
       this.getDehydratedIrisGridState(
@@ -1247,8 +1236,8 @@ export class IrisGridPanel extends PureComponent<
       model,
       movedColumns,
       movedRows,
-      partition,
-      partitionColumn,
+      partitions,
+      partitionColumns,
       quickFilters,
       reverseType,
       rollupConfig,
@@ -1329,8 +1318,8 @@ export class IrisGridPanel extends PureComponent<
             isStuckToRight={isStuckToRight}
             movedColumns={movedColumns}
             movedRows={movedRows}
-            partition={partition}
-            partitionColumn={partitionColumn}
+            partitions={partitions}
+            partitionColumns={partitionColumns}
             quickFilters={quickFilters}
             reverseType={reverseType}
             rollupConfig={rollupConfig}
@@ -1348,7 +1337,6 @@ export class IrisGridPanel extends PureComponent<
             onCreateChart={this.handleCreateChart}
             onDataSelected={this.handleDataSelected}
             onError={this.handleError}
-            onPartitionAppend={this.handlePartitionAppend}
             onStateChange={this.handleGridStateChange}
             onContextMenu={this.handleContextMenu}
             onAdvancedSettingsChange={this.handleAdvancedSettingsChange}