New approach for CSV reading

deephaven · Dec 23, 2021 · ea04752 · ea04752
1 parent 9161a57
commit ea04752
Show file tree

Hide file tree

Showing 85 changed files with 9,667 additions and 1,678 deletions.
diff --git a/ClientSupport/src/test/java/io/deephaven/treetable/SnapshotStateTest.java b/ClientSupport/src/test/java/io/deephaven/treetable/SnapshotStateTest.java
@@ -1,6 +1,7 @@
 package io.deephaven.treetable;
 
 import io.deephaven.csv.CsvTools;
+import io.deephaven.csv.util.CsvReaderException;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.lang.QueryLibrary;
 import io.deephaven.engine.util.TableTools;
@@ -9,7 +10,6 @@
 import io.deephaven.engine.table.impl.QueryTableTestBase;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.util.BitSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -18,7 +18,7 @@
 import static io.deephaven.treetable.TreeTableConstants.ROOT_TABLE_KEY;
 
 public class SnapshotStateTest extends QueryTableTestBase {
-    private static Table getRawNyMunis() throws IOException {
+    private static Table getRawNyMunis() throws CsvReaderException {
         QueryLibrary.importStatic(TreeSnapshotQueryTest.StaticHolder.class);
 
         final BaseTable base =
@@ -33,12 +33,12 @@ private static Table makeNyMunisTreeTableFrom(Table t) {
         return t.treeTable("Path", "Direct");
     }
 
-    private static Table makeNyMunisTreeTable() throws IOException {
+    private static Table makeNyMunisTreeTable() throws CsvReaderException {
         return makeNyMunisTreeTableFrom(getRawNyMunis());
     }
 
     @Test
-    public void testBounds() throws IOException {
+    public void testBounds() throws CsvReaderException {
         final HierarchicalTable treeTable = (HierarchicalTable) makeNyMunisTreeTable();
         final Map<Object, TableDetails> details = new HashMap<>();
 

diff --git a/ClientSupport/src/test/java/io/deephaven/treetable/TreeSnapshotQueryTest.java b/ClientSupport/src/test/java/io/deephaven/treetable/TreeSnapshotQueryTest.java
@@ -2,6 +2,7 @@
 
 import io.deephaven.base.Pair;
 import io.deephaven.csv.CsvTools;
+import io.deephaven.csv.util.CsvReaderException;
 import io.deephaven.datastructures.util.SmartKey;
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.Table;
@@ -21,7 +22,6 @@
 import gnu.trove.map.hash.TIntObjectHashMap;
 import gnu.trove.map.hash.TObjectIntHashMap;
 
-import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -183,7 +183,7 @@ public static List<String> removeEmpty(String... components) {
         }
     }
 
-    private static Table getRawNyMunis() throws IOException {
+    private static Table getRawNyMunis() throws CsvReaderException {
         QueryLibrary.importStatic(StaticHolder.class);
 
         final BaseTable base =
@@ -202,7 +202,7 @@ private static Table getRawNyMunis() throws IOException {
                 .lastBy("Path");
     }
 
-    private static Table makeNyMunisTreeTable() throws IOException {
+    private static Table makeNyMunisTreeTable() throws CsvReaderException {
         return makeNyMunisTreeTableFrom(getRawNyMunis());
     }
 
@@ -218,7 +218,7 @@ private static List<String> munisKey(String... path) {
 
     // region Actual Tests
 
-    public void testTsq() throws IOException {
+    public void testTsq() throws CsvReaderException {
         final Table t = makeNyMunisTreeTable();
         final TTState state = new TTState(t);
         final BitSet allColumns = new BitSet(t.getColumns().length);
@@ -456,7 +456,7 @@ public void testTsq() throws IOException {
         assertFalse(state.expansionMap.containsKey(mayfieldKey));
     }
 
-    public void testSortandFilter() throws IOException {
+    public void testSortandFilter() throws CsvReaderException {
         final Table t = makeNyMunisTreeTable();
         final TTState state = new TTState(t);
         final BitSet allColumns = new BitSet(t.getColumns().length);

diff --git a/Integrations/python/deephaven/csv.py b/Integrations/python/deephaven/csv.py
@@ -116,7 +116,7 @@ def read(path: str,
     Args:
         path (str): a file path or a URL string
         header (Dict[str, DataType]): a dict to define the table columns with key being the name, value being the data type
-        inference (csv.Inference): an Enum value specifying the rules for data type inference, default is INFERENCE_STANDARD_TIMES
+        inference (csv.Inference): an Enum value specifying the rules for data type inference, default is INFERENCE_STANDARD
         headless (bool): indicates if the CSV data is headless, default is False
         delimiter (str): the delimiter used by the CSV, default is the comma
         quote (str): the quote character for the CSV, default is double quote
@@ -133,7 +133,7 @@ def read(path: str,
     """
 
     if inference is None:
-        inference = INFERENCE_STANDARD_TIMES
+        inference = INFERENCE_STANDARD
 
     csv_specs_builder = _JCsvSpecs.builder()
 

diff --git a/Integrations/python/test/data/test_csv.csv b/Integrations/python/test/data/test_csv.csv
@@ -3,6 +3,6 @@ String,Long,Float
 a   ,9223372036854775807,
 b,,-Infinity
 c,-9223372036854775807,NaN
-"d 'c "  ,9999999,3.4028235e+38f
+"d 'c "  ,9999999,3.4028234e+38f
 null,-0,1.17549435E-38f
 "null",0,1.4e-45f
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/InMemoryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/InMemoryTable.java
@@ -39,6 +39,11 @@ public static InMemoryTable from(NewTable table) {
                 columns);
     }
 
+    public static InMemoryTable from(TableDefinition definition, TrackingRowSet rowSet,
+            Map<String, ? extends ColumnSource<?>> columns) {
+        return new InMemoryTable(definition, rowSet, columns);
+    }
+
     public InMemoryTable(String[] columnNames, Object[] arrayValues) {
         super(RowSetFactory.flat(Array.getLength(arrayValues[0])).toTracking(),
                 createColumnsMap(columnNames, arrayValues));