feat: Use specified branch of arrow-rs with workaround to invalid off…

…set buffers from Java Arrow
viirya · Mar 31, 2024 · e925ace · e925ace
1 parent aa6ddc5
commit e925ace
Show file tree

Hide file tree

Showing 14 changed files with 657 additions and 524 deletions.
diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala
@@ -133,11 +133,11 @@ object CometConf {
   val COMET_COLUMNAR_SHUFFLE_ENABLED: ConfigEntry[Boolean] = conf(
     "spark.comet.columnar.shuffle.enabled")
     .doc(
-      "Force Comet to only use columnar shuffle for CometScan and Spark regular operators. " +
-        "If this is enabled, Comet native shuffle will not be enabled but only Arrow shuffle. " +
-        "By default, this config is false.")
+      "Whether to enable Arrow-based columnar shuffle for Comet and Spark regular operators. " +
+        "If this is enabled, Comet prefers columnar shuffle than native shuffle. " +
+        "By default, this config is true.")
     .booleanConf
-    .createWithDefault(false)
+    .createWithDefault(true)
 
   val COMET_EXEC_BROADCAST_ENABLED: ConfigEntry[Boolean] =
     conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcast.enabled")