Add projection pushdown support to JsonRDD/JsonRDD2

zapletal-martin · May 1, 2015 · ab6ee87 · ab6ee87
1 parent f636c14
commit ab6ee87
Showing 1 changed file with 15 additions and 1 deletion.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -111,7 +111,8 @@ private[sql] class JSONRelation(
     @transient val sqlContext: SQLContext)
   extends BaseRelation
   with TableScan
-  with InsertableRelation {
+  with InsertableRelation
+  with CatalystScan {
 
   def this(
       path: String,
@@ -158,6 +159,19 @@ private[sql] class JSONRelation(
     }
   }
 
+  override def buildScan(requiredColumns: Seq[Attribute], filters: Seq[Expression]): RDD[Row] = {
+    if (useJsonRDD2) {
+      JsonRDD2.jsonStringToRow(
+        baseRDD,
+        StructType.fromAttributes(requiredColumns),
+        sqlContext.conf.columnNameOfCorruptRecord)
+    } else {
+      JsonRDD.jsonStringToRow(
+        baseRDD,
+        StructType.fromAttributes(requiredColumns),
+        sqlContext.conf.columnNameOfCorruptRecord)
+    }
+  }
 
   override def insert(data: DataFrame, overwrite: Boolean): Unit = {
     val filesystemPath = path match {