Merge pull request #25 from Sage-Bionetworks/add-snapshot-date

re-process fileinventory tables

Author: thomasyu888

synapse_data_warehouse/synapse_raw/tables/V2.12.0__add_snapshot_date_to_fileinventory.sql

@@ -0,0 +1,29 @@
+USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP
+ALTER TABLE fileinventory ADD COLUMN snapshot_date DATE;
+ALTER TABLE fileinventory CLUSTER BY (snapshot_date)
+
+USE WAREHOUSE COMPUTE_MEDIUM;
+TRUNCATE TABLE fileinventory;
+-- initial load of data
+copy into
+  fileinventory
+from (
+  select
+    $1:bucket as bucket,
+    $1:e_tag as e_tag,
+    $1:encryption_status as encryption_status,
+    $1:intelligent_tiering_access_tier as intelligent_tiering_access_tier,
+    $1:is_delete_marker as is_delete_marker,
+    $1:is_latest as is_latest,
+    $1:is_multipart_uploaded as is_multipart_uploaded,
+    $1:key as key,
+    $1:last_modified_date as last_modified_date,
+    $1:object_owner as object_owner,
+    $1:size as size,
+    $1:storage_class as storage_class,
+    metadata$file_last_modified as snapshot_date
+  from
+    @{{stage_storage_integration}}_stage/inventory --noqa: TMP
+  )
+pattern='.*defaultInventory/data/.*'
+;

synapse_data_warehouse/synapse_raw/tasks/V2.12.1__add_snapshot_to_fileinventory_task.sql

@@ -0,0 +1,33 @@
+use role accountadmin;
+use schema {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP
+alter task refresh_synapse_warehouse_s3_stage_task suspend;
+alter task append_to_fileinventory_task suspend;
+
+create task if not exists append_to_fileinventory_task
+    user_task_managed_initial_warehouse_size = 'SMALL'
+    AFTER refresh_synapse_warehouse_s3_stage_task
+as
+    copy into
+        fileinventory
+    from (
+        select
+            $1:bucket as bucket,
+            $1:e_tag as e_tag,
+            $1:encryption_status as encryption_status,
+            $1:intelligent_tiering_access_tier as intelligent_tiering_access_tier,
+            $1:is_delete_marker as is_delete_marker,
+            $1:is_latest as is_latest,
+            $1:is_multipart_uploaded as is_multipart_uploaded,
+            $1:key as key,
+            $1:last_modified_date as last_modified_date,
+            $1:object_owner as object_owner,
+            $1:size as size,
+            $1:storage_class as storage_class,
+            metadata$file_last_modified as snapshot_date
+        from
+            @{{stage_storage_integration}}_stage/inventory --noqa: TMP
+        )
+        pattern='.*defaultInventory/data/.*';
+
+alter task append_to_fileinventory_task resume;
+alter task refresh_synapse_warehouse_s3_stage_task resume;