EM: Lakeformation for Cloud platform roles

terraform/environments/electronic-monitoring-data/cloud_platform_share.tf

@@ -33,9 +33,10 @@ module "cmt_front_end_assumable_role" {
   source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
   version = "5.48.0"
 
-  trusted_role_arns = [
-    local.resolved-cloud-platform-iam-role
-  ]
+  trusted_role_arns = flatten([
+    local.resolved-cloud-platform-iam-role,
+    data.aws_iam_roles.data_engineering_roles.arns
+  ])
 
   create_role       = true
   role_requires_mfa = false
@@ -45,7 +46,27 @@ module "cmt_front_end_assumable_role" {
   tags = local.tags
 }
 
+module "specials_cmt_front_end_assumable_role" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+  source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
+  version = "5.48.0"
+
+  trusted_role_arns = flatten([
+    local.resolved-cloud-platform-iam-role,
+    data.aws_iam_roles.data_engineering_roles.arns
+  ])
+
+  create_role       = true
+  role_requires_mfa = false
+
+  role_name = "specials_cmt_read_emds_data_${local.environment_shorthand}"
+
+  tags = local.tags
+}
+
 # module "share_api_data_marts" {
+#   count = local.is-production ? 1 : 0
 #   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 #   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 #   source = "github.com/ministryofjustice/terraform-aws-analytical-platform-lakeformation?ref=32525da937012178e430585ac5a00f05193f58eb"
@@ -67,3 +88,77 @@ module "cmt_front_end_assumable_role" {
 #     aws.destination = aws
 #   }
 # }
+
+
+data "aws_iam_policy_document" "standard_athena_access" {
+  statement {
+    actions = [
+      "athena:GetDataCatalog",
+      "athena:GetQueryExecution",
+      "athena:GetQueryResults",
+      "athena:GetWorkGroup",
+      "athena:StartQueryExecution",
+      "athena:StopQueryExecution"
+    ]
+    resources = [
+      "arn:aws:athena:${data.aws_region.current.name}:${local.env_account_id}:*/*"
+    ]
+  }
+  statement {
+    actions = [
+      "athena:ListWorkGroups"
+    ]
+    resources = [
+      "*"
+    ]
+  }
+  statement {
+    actions   = ["lakeformation:GetDataAccess"]
+    resources = ["*"]
+  }
+  statement {
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:ListMultipartUploadParts"
+    ]
+    resources = [module.s3-athena-bucket.bucket.arn]
+  }
+  statement {
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:DeleteObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:ListMultipartUploadParts"
+    ]
+    resources = ["${module.s3-athena-bucket.bucket.arn}/*"]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "glue:GetDatabases",
+      "glue:GetDatabase",
+      "glue:GetTables",
+      "glue:GetTable"
+    ]
+    resources = [
+      "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:catalog",
+      "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:database/staged_fms_${local.env_}dbt",
+      "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:table/staged_fms_${local.env_}dbt/*"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "standard_athena_access" {
+  name_prefix = "standard_athena_access"
+  description = "Standard permissions for Athena"
+  policy      = data.aws_iam_policy_document.standard_athena_access.json
+}
+
+resource "aws_iam_role_policy_attachment" "standard_athena_access" {
+  policy_arn = aws_iam_policy.standard_athena_access.arn
+  role       = module.cmt_front_end_assumable_role.iam_role_name
+}

terraform/environments/electronic-monitoring-data/lake_formation_cloud_platform.tf

@@ -0,0 +1,69 @@
+locals {
+  env_          = "${local.environment_shorthand}_"
+  cap_dw_tables = local.is-production ? ["contact_history", "equipment_details", "event_history", "incident", "order_details", "services", "suspension_of_visits", "violations", "visit_details"] : []
+  am_tables     = local.is-production ? ["am_contact_history", "am_equipment_details", "am_incident", "am_order_details", "am_services", "am_visit_details"] : []
+}
+
+resource "aws_lakeformation_resource" "data_bucket" {
+  arn = module.s3-create-a-derived-table-bucket.bucket.arn
+}
+
+module "share_current_version" {
+  count  = local.is-test ? 1 : 0
+  source = "./modules/lakeformation"
+  table_filters = {
+    "account" = "__current=true"
+  }
+  role_arn                = module.cmt_front_end_assumable_role.iam_role_arn
+  database_name           = "staged_fms_${local.env_}dbt"
+  data_engineer_role_arn  = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+}
+
+module "cap_dw_excluding_specials" {
+  for_each = toset(local.cap_dw_tables)
+  source   = "./modules/lakeformation"
+  table_filters = {
+    (each.key) = "specials_flag=0"
+  }
+  role_arn                = module.cmt_front_end_assumable_role.iam_role_arn
+  database_name           = "historic_api_mart_historic_dev_dbt"
+  data_engineer_role_arn  = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+}
+
+module "cap_dw_including_specials" {
+  for_each = toset(local.cap_dw_tables)
+  source   = "./modules/lakeformation"
+  table_filters = {
+    (each.key) = ""
+  }
+  role_arn                = module.specials_cmt_front_end_assumable_role.iam_role_arn
+  database_name           = "historic_api_mart_historic_dev_dbt"
+  data_engineer_role_arn  = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+}
+
+module "am_for_non_specials_role" {
+  for_each = toset(local.am_tables)
+  source   = "./modules/lakeformation"
+  table_filters = {
+    (each.key) = ""
+  }
+  role_arn                = module.cmt_front_end_assumable_role.iam_role_arn
+  database_name           = "historic_api_mart_historic_dev_dbt"
+  data_engineer_role_arn  = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+}
+
+module "am_for_specials_role" {
+  for_each = toset(local.am_tables)
+  source   = "./modules/lakeformation"
+  table_filters = {
+    (each.key) = ""
+  }
+  role_arn                = module.specials_cmt_front_end_assumable_role.iam_role_arn
+  database_name           = "historic_api_mart_historic_dev_dbt"
+  data_engineer_role_arn  = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+}

terraform/environments/electronic-monitoring-data/modules/lakeformation/main.tf

@@ -0,0 +1,82 @@
+data "aws_caller_identity" "current" {}
+
+resource "aws_lakeformation_permissions" "data_engineering_permissions" {
+  permissions = ["ALL"]
+  principal   = var.data_engineer_role_arn
+
+  database {
+    name = var.database_name
+  }
+}
+
+resource "random_id" "suffix" {
+  byte_length = 32
+}
+
+resource "aws_lakeformation_permissions" "data_engineering_table_permissions" {
+  for_each    = var.table_filters
+  permissions = ["ALL"]
+  principal   = var.data_engineer_role_arn
+
+  table {
+    database_name = var.database_name
+    name          = each.key
+  }
+}
+
+resource "aws_lakeformation_permissions" "de_s3_bucket_permissions" {
+  principal = var.data_engineer_role_arn
+
+  permissions = ["DATA_LOCATION_ACCESS"]
+
+  data_location {
+    arn = var.data_bucket_lf_resource
+  }
+}
+
+resource "aws_lakeformation_permissions" "s3_bucket_permissions" {
+  principal = var.role_arn
+
+  permissions = ["DATA_LOCATION_ACCESS"]
+
+  data_location {
+    arn = var.data_bucket_lf_resource
+  }
+}
+
+resource "aws_lakeformation_data_cells_filter" "data_filter" {
+  for_each = tomap(var.table_filters)
+  table_data {
+    database_name    = var.database_name
+    name             = "filter-${each.key}-${random_id.suffix.hex}"
+    table_catalog_id = data.aws_caller_identity.current.account_id
+    table_name       = each.key
+    column_wildcard {
+      excluded_column_names = []
+    }
+    dynamic "row_filter" {
+      for_each = each.value != "" ? [each.value] : []
+      content {
+        filter_expression = each.value
+      }
+    }
+    dynamic "row_filter" {
+      for_each = each.value == "" ? [each.value] : []
+      content {
+        all_rows_wildcard {}
+      }
+    }
+  }
+}
+
+resource "aws_lakeformation_permissions" "share_filtered_data_with_role" {
+  for_each    = tomap(var.table_filters)
+  principal   = var.role_arn
+  permissions = ["DESCRIBE", "SELECT"]
+  data_cells_filter {
+    database_name    = var.database_name
+    table_name       = each.key
+    table_catalog_id = data.aws_caller_identity.current.account_id
+    name             = aws_lakeformation_data_cells_filter.data_filter[each.key].table_data[0].name
+  }
+}

terraform/environments/electronic-monitoring-data/modules/lakeformation/variables.tf

@@ -0,0 +1,25 @@
+variable "table_filters" {
+  description = "Map of names of the tables and filters to apply"
+  type        = map(string)
+}
+
+variable "database_name" {
+  description = "Name of the database the table belongs to"
+  type        = string
+
+}
+
+variable "data_engineer_role_arn" {
+  description = "ARN of the DE role"
+  type        = string
+}
+
+variable "data_bucket_lf_resource" {
+  description = "arn of the lake formation resource for the data bucket"
+  type        = string
+}
+
+variable "role_arn" {
+  description = "Role to grant permissions to"
+  type        = string
+}

terraform/environments/electronic-monitoring-data/modules/lakeformation/versions.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
+  }
+  required_version = ">= 1.0.1"
+}