diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala index 93f261b57..72d1ccb63 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/PipelineSparkSessionBuilder.scala @@ -64,7 +64,16 @@ object PipelineSparkSessionBuilder { case (builder, (key, value)) => builder.config(key, value) } - val spark = sparkSessionBuilderWithExtraOptApplied.getOrCreate() + val isHiveSupportRequired = extraOptions.exists(_._1.contains("hive")) + + val spark = if (isHiveSupportRequired) { + sparkSessionBuilderWithExtraOptApplied + .enableHiveSupport() + .getOrCreate() + } else { + sparkSessionBuilderWithExtraOptApplied + .getOrCreate() + } applyHadoopConfig(spark, conf) }