#139 Invoke .enableHiveSupport() when building the Spark session and …

…Hive options are specified.
AbsaOSS · Jan 26, 2023 · 6200fae · 6200fae
1 parent b67de4f
commit 6200fae
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala
@@ -64,7 +64,16 @@ object PipelineSparkSessionBuilder {
       case (builder, (key, value)) => builder.config(key, value)
     }
 
-    val spark = sparkSessionBuilderWithExtraOptApplied.getOrCreate()
+    val isHiveSupportRequired = extraOptions.exists(_._1.contains("hive"))
+
+    val spark = if (isHiveSupportRequired) {
+      sparkSessionBuilderWithExtraOptApplied
+        .enableHiveSupport()
+        .getOrCreate()
+    } else {
+      sparkSessionBuilderWithExtraOptApplied
+        .getOrCreate()
+    }
 
     applyHadoopConfig(spark, conf)
   }