DPR2-1642: Include part-*.snappy.parquet in batch job during ingestio…

…n replay
ministryofjustice · Dec 30, 2024 · 9e9d3e6 · 9e9d3e6
1 parent cc490c5
commit 9e9d3e6
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf b/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf
@@ -166,6 +166,7 @@ module "replay_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_reporting_hub_batch_jobname,
             "Arguments" : {
+              "--dpr.batch.load.fileglobpattern" : "{part-*.snappy.parquet,LOAD*parquet}",
               "--dpr.config.s3.bucket" : var.s3_glue_bucket_id,
               "--dpr.config.key" : var.domain
             }