Fix for issue #70

deephaven · Oct 8, 2022 · 05a5fb4 · 05a5fb4
1 parent fabda31
commit 05a5fb4
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 1 deletion.
diff --git a/src/main/java/io/deephaven/csv/tokenization/Tokenizer.java b/src/main/java/io/deephaven/csv/tokenization/Tokenizer.java
@@ -83,6 +83,7 @@ public boolean tryParseBMPChar(final ByteSlice bs, final MutableInt result) {
             // Last code point U+007F
             value = first & 0x7F;
             result.setValue(value);
+            // Succeed only if the string ended here.
             return o == end;
         }
         if ((first & 0xE0) == 0xC0) {
@@ -117,7 +118,8 @@ public boolean tryParseBMPChar(final ByteSlice bs, final MutableInt result) {
         }
 
         result.setValue(value);
-        return true;
+        // Succeed only if the string ended here.
+        return o == end;
     }
 
     private static int byteToInt(byte b) {

diff --git a/src/test/java/io/deephaven/csv/CsvReaderTest.java b/src/test/java/io/deephaven/csv/CsvReaderTest.java
@@ -110,6 +110,27 @@ public void bug52() throws CsvReaderException {
         CsvReader.read(specs, inputStream, sf);
     }
 
+    /**
+     * Reported in <a href="https://github.com/deephaven/deephaven-core/issues/2898">Deephaven Core Issue #2898</a>.
+     * Bug filed in <a href="https://github.com/deephaven/deephaven-csv/issues/70">Deephaven CSV Issue #70</a>.
+     */
+    @Test
+    public void bug70() throws CsvReaderException {
+        final String input = "Coin,Change,Remark\r\n" +
+                "USDT,-49.00787612,\r\n" +
+                "USDT,-152.686844,穿仓保证金补偿\r\n" +
+                "USDT,-59.92650232,\r\n" +
+                "USDT,-102.3862566,\r\n";
+
+        final ColumnSet expected =
+                ColumnSet.of(
+                        Column.ofRefs("Coin", "USDT", "USDT", "USDT", "USDT"),
+                        Column.ofValues("Change", -49.00787612, -152.686844, -59.92650232, -102.3862566),
+                        Column.ofRefs("Remark", null, "穿仓保证金补偿", null, null)
+                        );
+        invokeTest(defaultCsvBuilder().parsers(Parsers.DEFAULT).build(), input, expected);
+    }
+
     @Test
     public void validates() {
         final String lengthyMessage = "CsvSpecs failed validation for the following reasons: "