[HOTFIX]: bug in DynamicIntArray.toArray() and redundant memory copy …

…for HeapByteBuffer in StringColumnReader. (#57)
pixelsdb · Jan 8, 2020 · d57b65a · d57b65a
1 parent 4796faf
commit d57b65a
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 5 deletions.
diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/StringColumnReader.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/StringColumnReader.java
@@ -239,11 +239,16 @@ private void readContent(int inputLength, PixelsProto.ColumnEncoding encoding)
             inputBuffer.resetReaderIndex();
             // read buffers
             contentBuf = inputBuffer.slice(0, originsOffset);
+            if (this.inputBuffer.isDirect())
             {
                 byte[] bytes = new byte[startsOffset - originsOffset];
                 inputBuffer.getBytes(originsOffset, bytes, 0, startsOffset - originsOffset);
                 originsBuf = Unpooled.wrappedBuffer(bytes);
             }
+            else
+            {
+                originsBuf = inputBuffer.slice(originsOffset, startsOffset - originsOffset);
+            }
             // read starts and orders
             ByteBuf startsBuf = inputBuffer.slice(startsOffset, ordersOffset - startsOffset);
             ByteBuf ordersBuf = inputBuffer.slice(ordersOffset, inputLength - ordersOffset);
@@ -271,11 +276,16 @@ private void readContent(int inputLength, PixelsProto.ColumnEncoding encoding)
             int lensOffset = inputBuffer.readInt();
             inputBuffer.resetReaderIndex();
             // read strings
+            if (this.inputBuffer.isDirect())
             {
                 byte[] bytes = new byte[lensOffset];
                 inputBuffer.getBytes(0, bytes, 0, lensOffset);
                 contentBuf = Unpooled.wrappedBuffer(bytes);
             }
+            else
+            {
+                contentBuf = inputBuffer.slice(0, lensOffset);
+            }
             // read lens field
             ByteBuf lensBuf = inputBuffer.slice(lensOffset, inputLength - Integer.BYTES - lensOffset);
             lensDecoder = new RunLenIntDecoder(new ByteBufInputStream(lensBuf), false);

diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/utils/DynamicIntArray.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/utils/DynamicIntArray.java
@@ -42,7 +42,7 @@ public final class DynamicIntArray
 
     private final int chunkSize;       // our allocation size
     private int[][] data;              // the real data
-    private int length;                // max set element index +1
+    private int length = 0;                // max set element index +1
     private int initializedChunks = 0; // the number of created chunks
 
     public DynamicIntArray()
@@ -175,15 +175,18 @@ public int[] toArray()
         {
             int[] array = new int[length];
             int i;
-            for (i = 0; i < initializedChunks-1; i++)
+            int numChunks = length / chunkSize;
+            for (i = 0; i < numChunks; i++)
             {
                 System.arraycopy(data[i], 0, array, i*chunkSize, chunkSize);
             }
+
             int tail = length % chunkSize;
-            for (int j = 0, k = i*chunkSize; j < tail; ++j, ++k)
+            if (tail > 0)
             {
-                array[k] = data[i][j];
+                System.arraycopy(data[i], 0, array, i*chunkSize, tail);
             }
+
             return array;
         }
     }

diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestDynamicIntArray.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/utils/TestDynamicIntArray.java
@@ -0,0 +1,25 @@
+package io.pixelsdb.pixels.core.utils;
+
+import org.junit.Test;
+
+/**
+ * Created at: 20-1-8
+ * Author: hank
+ */
+public class TestDynamicIntArray
+{
+    @Test
+    public void test ()
+    {
+        DynamicIntArray array = new DynamicIntArray(64);
+        for (int i = 0; i < 1024; ++i)
+        {
+            array.add(i);
+        }
+        int[] ints = array.toArray();
+        for (int i = 0; i < array.size(); ++i)
+        {
+            assert i == ints[i];
+        }
+    }
+}
diff --git a/pixels-load/src/main/java/io/pixelsdb/pixels/load/multi/Main.java b/pixels-load/src/main/java/io/pixelsdb/pixels/load/multi/Main.java
@@ -67,7 +67,7 @@
  *
  * <br>This shall be run under root user to execute cache cleaning commands
  * <p>
- * QUERY -t pixels -w /home/iir/opt/pixels/1187_dedup_query.txt -l /home/iir/opt/pixels/pixels_duration_1187_v_1_compact_cache.csv -c /home/iir/opt/presto-server/sbin/drop-caches.sh
+ * QUERY -t pixels -w /home/iir/opt/pixels/1187_dedup_query.txt -l /home/iir/opt/pixels/pixels_duration_1187_v_1_compact_cache_2020.01.09.csv -c /home/iir/opt/presto-server/sbin/drop-caches.sh
  * </p>
  * <p> Local
  * QUERY -t pixels -w /home/tao/software/station/bitbucket/105_dedup_query.txt -l /home/tao/software/station/bitbucket/pixels_duration_local.csv