Add Lake Formation sharing module for analytical platform data

terraform/environments/electronic-monitoring-data/analytical_platform_lf_share.tf

@@ -0,0 +1,32 @@
+locals {
+  data_locations = [
+    {
+      data_location = module.s3-create-a-derived-table-bucket.bucket.arn
+      hybrid_access = true
+      register      = true
+      share         = true
+
+    }
+  ]
+
+  databases = [
+    {
+      source_database  = "staged_fms_${local.env_}dbt"
+      source_table     = "account"
+      permissions      = ["SELECT"]
+      row_filter       = "__current=true"
+      excluded_columns = []
+    }
+  ]
+}
+
+module "analytical_platform_lf_share" {
+  count  = local.is-test ? 1 : 0
+  source = "./modules/analytical-platform-lakeformation"
+
+  destination_account_id = local.environment_management.account_ids["analytical-platform-data-engineering-sandboxa"]
+
+  data_locations = local.data_locations
+
+  databases_to_share = local.databases
+}

terraform/environments/electronic-monitoring-data/analytical_platform_share.tf

@@ -323,7 +323,8 @@ data "aws_iam_policy_document" "analytical_platform_share_policy" {
       "lakeformation:DeregisterResource",
       "lakeformation:ListPermissions",
       "lakeformation:DescribeResource",
-
+      "lakeformation:CreateDataCellsFilter",
+      "lakeformation:GetDataCellsFilter",
     ]
     resources = [
       #checkov:skip=CKV_AWS_356: "Ensure no IAM policies documents allow "*" as a statement's resource for restrictable actions"
@@ -393,7 +394,7 @@ resource "aws_iam_role" "analytical_platform_share_role" {
         Effect = "Allow"
         Principal = {
           # In case consumer has a central location for terraform state storage that isn't the target account.
-          AWS = "arn:aws:iam::${try(each.value.assume_account_id, each.value.target_account_id)}:root"
+          AWS = "arn:aws:iam::${local.environment_management.account_ids["analytical-platform-data-engineering-sandboxa"]}:root"
         }
         Action = "sts:AssumeRole"
       }

terraform/environments/electronic-monitoring-data/application_variables.json

@@ -14,8 +14,10 @@
           ],
           "resource_shares": [
             {
-              "glue_database": "dms_data_validation",
-              "glue_tables": ["*"]
+              "glue_database": "serco_fms_test_dbt",
+              "glue_tables": [
+                "*"
+              ]
             }
           ]
         }
@@ -27,15 +29,19 @@
       "enable_dbt_k8s_secrets": true,
       "analytical_platform_share": [
         {
-          "target_account_name": "analytical-platform-data-production",
-          "target_account_id": "593291632749",
+          "target_account_name": "analytical-platform-data-engineering-sandbox-a",
+          "target_account_id": "684969100054",
           "assume_account_name": "analytical-platform-management-production",
           "assume_account_id": "042130406152",
-          "data_locations": ["emds-test-bucket-name"],
+          "data_locations": [
+            "emds-test-bucket-name"
+          ],
           "resource_shares": [
             {
-              "glue_database": "test_db_name",
-              "glue_tables": ["*"]
+              "glue_database": "staged_fms_test_dbt",
+              "glue_tables": [
+                "*"
+              ]
             }
           ]
         }
@@ -56,16 +62,20 @@
           "target_account_id": "593291632749",
           "assume_account_name": "analytical-platform-management-production",
           "assume_account_id": "042130406152",
-          "data_locations": ["emds-prod-bucket-name"],
+          "data_locations": [
+            "emds-prod-bucket-name"
+          ],
           "resource_shares": [
             {
               "glue_database": "capita_alcohol_monitoring",
-              "glue_tables": ["*"]
+              "glue_tables": [
+                "*"
+              ]
             }
           ]
         }
       ],
       "enable_airflow_secret": true
     }
   }
-}
+}

terraform/environments/electronic-monitoring-data/modules/analytical-platform-lakeformation/main.tf

@@ -0,0 +1,61 @@
+data "aws_caller_identity" "current" {}
+
+resource "aws_lakeformation_resource" "data_location" {
+  for_each                = { for idx, loc in var.data_locations : loc.data_location => loc }
+  arn                     = each.value.data_location
+  use_service_linked_role = true
+}
+
+resource "aws_lakeformation_permissions" "data_location_share" {
+  for_each  = { for idx, loc in var.data_locations : loc.data_location => loc }
+  principal = var.destination_account_id
+
+  permissions = ["DATA_LOCATION_ACCESS"]
+
+  data_location {
+    arn = each.value.data_location
+  }
+}
+
+
+resource "aws_lakeformation_data_cells_filter" "data_filter" {
+  for_each = {
+    for db in var.databases_to_share : db.source_database => db
+  }
+  table_data {
+    database_name    = each.key
+    name             = "filter-${each.value.source_table}"
+    table_catalog_id = data.aws_caller_identity.current.account_id
+    table_name       = each.value.source_table
+    column_wildcard {
+      excluded_column_names = each.value.excluded_columns
+    }
+    dynamic "row_filter" {
+      for_each = each.value.row_filter != "" ? [each.value.row_filter] : []
+      content {
+        filter_expression = each.value.row_filter
+      }
+    }
+    dynamic "row_filter" {
+      for_each = each.value.row_filter == "" ? [each.value.row_filter] : []
+      content {
+        all_rows_wildcard {}
+      }
+    }
+  }
+}
+
+resource "aws_lakeformation_permissions" "share_filtered_data" {
+  for_each = {
+    for db in var.databases_to_share : db.source_database => db
+  }
+  permissions = each.value.permissions
+  principal   = var.destination_account_id
+
+  data_cells_filter {
+    database_name    = each.key
+    table_name       = each.value.source_table
+    table_catalog_id = data.aws_caller_identity.current.account_id
+    name             = aws_lakeformation_data_cells_filter.data_filter[each.key].table_data[0].name
+  }
+}

terraform/environments/electronic-monitoring-data/modules/analytical-platform-lakeformation/variables.tf

@@ -0,0 +1,22 @@
+variable "destination_account_id" {
+  description = "The account ID of the destination account"
+  type        = string
+}
+
+variable "data_locations" {
+  description = "A list of data locations to share"
+  type = list(object({
+    data_location = string
+  }))
+}
+
+variable "databases_to_share" {
+  description = "A list of databases to share"
+  type = list(object({
+    source_database  = string
+    source_table     = string
+    row_filter       = string
+    permissions      = list(string)
+    excluded_columns = list(string)
+  }))
+}

terraform/environments/electronic-monitoring-data/modules/analytical-platform-lakeformation/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+  required_version = ">= 1.0.1"
+}