From f65794b916237c526546e64520c25fe2a966a65b Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Fri, 13 Dec 2024 14:51:57 +0100 Subject: [PATCH] #520 Add documentation for the identity transformer. --- .../transformers/IdentityTransformer.scala | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala index 6e63ebb61..343cb5e88 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala @@ -22,6 +22,40 @@ import za.co.absa.pramen.core.transformers.IdentityTransformer._ import java.time.LocalDate +/** + * The transformer does not do any actual transformation and just returns the input DataFrame. + * + * It can be used to copy data between metastore tables located in different storages. + * + * The transformer supports incremental processing. + * + * Example usage: + * {{{ + * pramen.operations = [ + * { + * name = "Copy table" + * type = "transformation" + * + * class = "za.co.absa.pramen.core.transformers.IdentityTransformer" + * schedule.type = "daily" + * + * dependencies = [ + * { + * tables = [ table_from ] + * date.from = "@infoDate" + * } + * ] + * + * option { + * input.table = "table_from" + * empty.allowed = true + * } + * + * output.table = "table_to" + * } + * ] + * }}} + */ class IdentityTransformer extends Transformer { override def validate(metastore: MetastoreReader, infoDate: LocalDate, options: Map[String, String]): Reason = { if (!options.contains(INPUT_TABLE_KEY) && !options.contains(INPUT_TABLE_LEGACY_KEY)) { @@ -54,4 +88,4 @@ object IdentityTransformer { val INPUT_TABLE_KEY = "input.table" val INPUT_TABLE_LEGACY_KEY = "table" val EMPTY_ALLOWED_KEY = "empty.allowed" -} \ No newline at end of file +}