#520 Increase the size of offset table names, and various fixups.

The table name in 'offsets' table should be bigger than in bookkeeping because there are virtual tables that take other table names as part of it.
AbsaOSS · Dec 10, 2024 · 5bb749b · 5bb749b
1 parent 7ff08a4
commit 5bb749b
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 5 deletions.
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecords.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecords.scala
@@ -19,7 +19,7 @@ package za.co.absa.pramen.core.bookkeeper.model
 import slick.jdbc.H2Profile.api._
 
 class OffsetRecords(tag: Tag) extends Table[OffsetRecord](tag, "offsets") {
-  def pramenTableName = column[String]("table_name", O.Length(200))
+  def pramenTableName = column[String]("table_name", O.Length(256))
   def infoDate = column[String]("info_date", O.Length(20))
   def dataType = column[String]("data_type", O.Length(20))
 

diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sink/LocalCsvSink.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sink/LocalCsvSink.scala
@@ -24,9 +24,8 @@ import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api.{ExternalChannelFactory, MetastoreReader, Sink, SinkResult}
 import za.co.absa.pramen.core.sink.LocalCsvSink.OUTPUT_PATH_KEY
-import za.co.absa.pramen.core.utils.{FsUtils, LocalFsUtils}
+import za.co.absa.pramen.core.utils.FsUtils
 
-import java.io.{BufferedWriter, FileOutputStream, OutputStreamWriter}
 import java.nio.file.{Files, Paths}
 import java.time.format.DateTimeFormatter
 import java.time.{LocalDate, ZonedDateTime}
@@ -82,7 +81,7 @@ import java.time.{LocalDate, ZonedDateTime}
   *
   *    tables = [
   *      {
-  *        metastore.table = metastore_table
+  *        input.metastore.table = metastore_table
   *        output.path = "/local/csv/path"
   *
   *        # Date range to read the source table for. By default the job information date is used.

diff --git a/pramen/extras/src/main/scala/za/co/absa/pramen/extras/sink/KafkaAvroSink.scala b/pramen/extras/src/main/scala/za/co/absa/pramen/extras/sink/KafkaAvroSink.scala
@@ -83,7 +83,7 @@ import java.time.LocalDate
   *
   *    tables = [
   *      {
-  *        metastore.table = metastore_table
+  *        input.metastore.table = metastore_table
   *        output.topic.name = "my.topic"
   *
   *        # All following settings are OPTIONAL