CR

apache · Mar 4, 2016 · 3450313 · 3450313
1 parent f35394c
commit 3450313
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 20 deletions.
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
@@ -71,7 +71,9 @@ public static void populate(ColumnVector col, InternalRow row, int fieldIdx) {
       } else if (t instanceof DecimalType) {
         DecimalType dt = (DecimalType)t;
         Decimal d = row.getDecimal(fieldIdx, dt.precision(), dt.scale());
-        if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
+        if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
+          col.putInts(0, capacity, (int)d.toUnscaledLong());
+        } else if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
           col.putLongs(0, capacity, d.toUnscaledLong());
         } else {
           final BigInteger integer = d.toJavaBigDecimal().unscaledValue();

diff --git a/.../test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadBenchmark.scala b/.../test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadBenchmark.scala
@@ -157,19 +157,19 @@ object ParquetReadBenchmark {
         Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz
         SQL Single Int Column Scan:         Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative
         -------------------------------------------------------------------------------------------
-        SQL Parquet Vectorized                    657 /  778         23.9          41.8       1.0X
-        SQL Parquet MR                           1606 / 1731          9.8         102.1       0.4X
-        SQL Parquet Non-Vectorized               1133 / 1216         13.9          72.1       0.6X
+        SQL Parquet Vectorized                    215 /  262         73.0          13.7       1.0X
+        SQL Parquet MR                           1946 / 2083          8.1         123.7       0.1X
+        SQL Parquet Non-Vectorized               1079 / 1213         14.6          68.6       0.2X
         */
         sqlBenchmark.run()
 
         /*
         Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz
         Parquet Reader Single Int Column    Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative
         -------------------------------------------------------------------------------------------
-        ParquetReader (Non-vectorized)            565 /  609         27.8          35.9       1.0X
-        ParquetReader(Batched)                    165 /  174         95.3          10.5       3.4X
-        ParquetReader(Batch -> Row)               158 /  188         99.3          10.1       3.6X
+        ParquetReader Non-Vectorized              610 /  737         25.8          38.8       1.0X
+        ParquetReader Vectorized                  123 /  152        127.8           7.8       5.0X
+        ParquetReader Vectorized -> Row           165 /  180         95.2          10.5       3.7X
         */
         parquetReaderBenchmark.run()
       }
@@ -222,10 +222,10 @@ object ParquetReadBenchmark {
         Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz
         Int and String Scan:                Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative
         -------------------------------------------------------------------------------------------
-        SQL Parquet Vectorized                   1025 / 1180         10.2          97.8       1.0X
-        SQL Parquet MR                           2157 / 2222          4.9         205.7       0.5X
-        SQL Parquet Non-vectorized               1450 / 1466          7.2         138.3       0.7X
-        ParquetReader Non-vectorized             1005 / 1022         10.4          95.9       1.0X
+        SQL Parquet Vectorized                    628 /  720         16.7          59.9       1.0X
+        SQL Parquet MR                           1905 / 2239          5.5         181.7       0.3X
+        SQL Parquet Non-vectorized               1429 / 1732          7.3         136.3       0.4X
+        ParquetReader Non-vectorized              989 / 1357         10.6          94.3       0.6X
         */
         benchmark.run()
       }
@@ -256,8 +256,8 @@ object ParquetReadBenchmark {
         Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz
         String Dictionary:                  Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative
         -------------------------------------------------------------------------------------------
-        SQL Parquet Vectorized                    578 /  593         18.1          55.1       1.0X
-        SQL Parquet MR                           1021 / 1032         10.3          97.4       0.6X
+        SQL Parquet Vectorized                    329 /  337         31.9          31.4       1.0X
+        SQL Parquet MR                           1131 / 1325          9.3         107.8       0.3X
         */
         benchmark.run()
       }
@@ -290,20 +290,19 @@ object ParquetReadBenchmark {
         Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz
         Partitioned Table:                  Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative
         -------------------------------------------------------------------------------------------
-        Read data column                          751 /  805         20.9          47.8       1.0X
-        Read partition column                     713 /  761         22.1          45.3       1.1X
-        Read both columns                        1004 / 1109         15.7          63.8       0.7X
+        Read data column                          191 /  250         82.1          12.2       1.0X
+        Read partition column                      82 /   86        192.4           5.2       2.3X
+        Read both columns                         220 /  248         71.5          14.0       0.9X
          */
         benchmark.run()
       }
     }
   }
 
   def main(args: Array[String]): Unit = {
-    //intScanBenchmark(1024 * 1024 * 15)
-    //intStringScanBenchmark(1024 * 1024 * 10)
-    //stringDictionaryScanBenchmark(1024 * 1024 * 10)
+    intScanBenchmark(1024 * 1024 * 15)
+    intStringScanBenchmark(1024 * 1024 * 10)
+    stringDictionaryScanBenchmark(1024 * 1024 * 10)
     partitionTableScanBenchmark(1024 * 1024 * 15)
-
   }
 }