#162 Remove redundant creation of directories that will be created an…

…yway.
AbsaOSS · Mar 7, 2023 · da2c16d · da2c16d
1 parent 4b7b4b5
commit da2c16d
Showing 1 changed file with 2 additions and 8 deletions.
diff --git a/pramen/extras/src/main/scala/za/co/absa/pramen/extras/sink/StandardizationSink.scala b/pramen/extras/src/main/scala/za/co/absa/pramen/extras/sink/StandardizationSink.scala
@@ -219,27 +219,21 @@ class StandardizationSink(sinkConfig: Config,
 
   private[extras] def writeToRawFolder(df: DataFrame,
                                        recordCount: Long,
-                                       outputPartitionPath: Path)(implicit spark: SparkSession): Unit = {
+                                       outputPartitionPath: Path): Unit = {
     val outputPathStr = outputPartitionPath.toUri.toString
     log.info(s"Saving $recordCount records to the Enceladus raw folder: $outputPathStr")
 
-    val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, outputPathStr)
-    fsUtils.createDirectoryRecursiveButLast(outputPartitionPath)
-
     df.write
       .mode(SaveMode.Overwrite)
       .json(outputPathStr)
   }
 
   private[extras] def writeToPublishFolder(df: DataFrame,
                                            recordCount: Long,
-                                           outputPartitionPath: Path)(implicit spark: SparkSession): Unit = {
+                                           outputPartitionPath: Path): Unit = {
     val outputPathStr = outputPartitionPath.toUri.toString
     log.info(s"Saving $recordCount records to the Enceladus publish folder: $outputPathStr")
 
-    val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, outputPathStr)
-    fsUtils.createDirectoryRecursiveButLast(outputPartitionPath)
-
     df.write
       .mode(SaveMode.Overwrite)
       .parquet(outputPathStr)