🚚 Add egress functionality to Analytical Platform Ingestion

terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf

@@ -1,4 +1,6 @@
 module "transfer_structured_logs" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/cloudwatch/aws//modules/log-group"
   version = "5.3.1"
 

terraform/environments/analytical-platform-ingestion/data.tf

@@ -1,2 +1 @@
-#### This file can be used to store data specific to the member account ####
 data "aws_availability_zones" "available" {}

terraform/environments/analytical-platform-ingestion/environment-configuration.tf

@@ -20,15 +20,14 @@ locals {
       target_buckets = ["dev-ingestion-testing"]
 
       /* Transfer Server */
-      transfer_server_hostname = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
-      transfer_server_sftp_users = {
-        "jacobwoffenden" = {
-          ssh_key     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
-          cidr_blocks = ["90.246.52.170/32"]
-        },
-        "garyhenderson" = {
-          ssh_key     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID2lrI7AhZ9Sy/JAVDfPPEkCZawuuVJ7MHg6NNAwYImb"
-          cidr_blocks = ["154.47.111.68/32"]
+      transfer_server_hostname   = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
+      transfer_server_sftp_users = {}
+      transfer_server_sftp_users_with_egress = {
+        "jacobwoffenden-egress" = {
+          ssh_key               = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
+          cidr_blocks           = ["90.246.52.170/32"]
+          egress_bucket         = module.bold_egress_bucket.s3_bucket_id
+          egress_bucket_kms_key = module.s3_bold_egress_kms.key_arn
         }
       }
     }
@@ -51,17 +50,9 @@ locals {
       target_buckets = ["dev-ingestion-testing"]
 
       /* Transfer Server */
-      transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
-      transfer_server_sftp_users = {
-        "jacobwoffenden" = {
-          ssh_key     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
-          cidr_blocks = ["90.246.52.170/32"]
-        },
-        "garyhenderson" = {
-          ssh_key     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID2lrI7AhZ9Sy/JAVDfPPEkCZawuuVJ7MHg6NNAwYImb"
-          cidr_blocks = ["154.47.111.68/32"]
-        }
-      }
+      transfer_server_hostname               = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
+      transfer_server_sftp_users             = {}
+      transfer_server_sftp_users_with_egress = {}
     }
   }
 }

terraform/environments/analytical-platform-ingestion/kms-keys.tf

@@ -1,5 +1,6 @@
 module "transfer_logs_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -39,6 +40,7 @@ module "transfer_logs_kms" {
 
 module "s3_landing_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -51,6 +53,7 @@ module "s3_landing_kms" {
 
 module "s3_processed_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -63,6 +66,7 @@ module "s3_processed_kms" {
 
 module "s3_quarantine_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -75,6 +79,7 @@ module "s3_quarantine_kms" {
 
 module "s3_definitions_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -87,6 +92,7 @@ module "s3_definitions_kms" {
 
 module "s3_bold_egress_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -115,6 +121,7 @@ module "s3_bold_egress_kms" {
 
 module "sns_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -127,6 +134,7 @@ module "sns_kms" {
 
 module "govuk_notify_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 
@@ -139,6 +147,7 @@ module "govuk_notify_kms" {
 
 module "supplier_data_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/kms/aws"
   version = "2.2.1"
 

terraform/environments/analytical-platform-ingestion/lambda-functions.tf

@@ -1,5 +1,6 @@
 module "definition_upload_lambda" {
   #checkov:skip=CKV_TF_1:Module is from Terraform registry
+
   source  = "terraform-aws-modules/lambda/aws"
   version = "7.2.1"
 
@@ -53,6 +54,7 @@ module "definition_upload_lambda" {
 
 module "scan_lambda" {
   #checkov:skip=CKV_TF_1:Module is from Terraform registry
+
   source  = "terraform-aws-modules/lambda/aws"
   version = "7.2.1"
 
@@ -123,6 +125,7 @@ module "scan_lambda" {
 
 module "transfer_lambda" {
   #checkov:skip=CKV_TF_1:Module is from Terraform registry
+
   source  = "terraform-aws-modules/lambda/aws"
   version = "7.2.1"
 

terraform/environments/analytical-platform-ingestion/modules/transfer-family/user-with-egress/main.tf

@@ -0,0 +1,108 @@
+data "aws_iam_policy_document" "this" {
+  statement {
+    sid    = "AllowKMS"
+    effect = "Allow"
+    actions = [
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Encrypt",
+      "kms:DescribeKey",
+      "kms:Decrypt",
+    ]
+    resources = [
+      var.landing_bucket_kms_key,
+      var.egress_bucket_kms_key
+    ]
+  }
+  statement {
+    sid     = "AllowS3ListBucket"
+    effect  = "Allow"
+    actions = ["s3:ListBucket"]
+    resources = [
+      "arn:aws:s3:::${var.landing_bucket}",
+      "arn:aws:s3:::${var.egress_bucket}"
+    ]
+  }
+  statement {
+    sid       = "AllowS3LandingBucketObjectActions"
+    effect    = "Allow"
+    actions   = ["s3:PutObject"]
+    resources = ["arn:aws:s3:::${var.landing_bucket}/${var.name}/*"]
+  }
+  statement {
+    sid    = "AllowS3EgressBucketObjectActions"
+    effect = "Allow"
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectAcl",
+      "s3:GetObjectVersion"
+    ]
+    resources = ["arn:aws:s3:::${var.egress_bucket}/${var.name}/*"]
+  }
+}
+
+module "policy" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
+  version = "5.37.1"
+
+  name_prefix = "transfer-user-${var.name}"
+
+  policy = data.aws_iam_policy_document.this.json
+}
+
+module "role" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
+  version = "5.37.1"
+
+  create_role = true
+
+  role_name         = "transfer-user-${var.name}"
+  role_requires_mfa = false
+
+  trusted_role_services = ["transfer.amazonaws.com"]
+
+  custom_role_policy_arns = [module.policy.arn]
+}
+
+resource "aws_transfer_user" "this" {
+  server_id = var.transfer_server
+  user_name = var.name
+  role      = module.role.iam_role_arn
+
+  home_directory_type = "LOGICAL"
+  home_directory_mappings {
+    entry  = "/upload"
+    target = "/${var.landing_bucket}/${var.name}"
+  }
+
+  home_directory_mappings {
+    entry  = "/download"
+    target = "/${var.egress_bucket}/${var.name}"
+  }
+}
+
+resource "aws_transfer_ssh_key" "this" {
+  server_id = var.transfer_server
+  user_name = aws_transfer_user.this.user_name
+  body      = var.ssh_key
+}
+
+resource "aws_security_group_rule" "this" {
+  type              = "ingress"
+  from_port         = 2222
+  to_port           = 2222
+  protocol          = "tcp"
+  cidr_blocks       = var.cidr_blocks
+  security_group_id = var.transfer_server_security_group
+}
+
+resource "aws_secretsmanager_secret" "this" {
+  for_each = toset(["technical-contact", "data-contact", "target-bucket"])
+
+  name       = "ingestion/sftp/${var.name}/${each.key}"
+  kms_key_id = var.supplier_data_kms_key
+}

terraform/environments/analytical-platform-ingestion/modules/transfer-family/user-with-egress/variables.tf

@@ -0,0 +1,39 @@
+variable "name" {
+  type = string
+}
+
+variable "ssh_key" {
+  type = string
+}
+
+variable "cidr_blocks" {
+  type = list(string)
+}
+
+variable "transfer_server" {
+  type = string
+}
+
+variable "transfer_server_security_group" {
+  type = string
+}
+
+variable "landing_bucket" {
+  type = string
+}
+
+variable "landing_bucket_kms_key" {
+  type = string
+}
+
+variable "egress_bucket" {
+  type = string
+}
+
+variable "egress_bucket_kms_key" {
+  type = string
+}
+
+variable "supplier_data_kms_key" {
+  type = string
+}

terraform/environments/analytical-platform-ingestion/modules/transfer-family/user/main.tf

@@ -11,7 +11,6 @@ data "aws_iam_policy_document" "this" {
     ]
     resources = [var.landing_bucket_kms_key]
   }
-  # TODO: review the permissions
   statement {
     sid     = "AllowS3ListBucket"
     effect  = "Allow"
@@ -21,17 +20,17 @@ data "aws_iam_policy_document" "this" {
       "arn:aws:s3:::${var.landing_bucket}/${var.name}/*"
     ]
   }
-  # TODO: review the permissions
   statement {
-    sid       = "AllowS3ObjectActions"
+    sid       = "AllowS3LandingBucketObjectActions"
     effect    = "Allow"
-    actions   = ["s3:*"]
+    actions   = ["s3:PutObject"]
     resources = ["arn:aws:s3:::${var.landing_bucket}/${var.name}/*"]
   }
 }
 
 module "policy" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
   version = "5.37.1"
 
@@ -57,24 +56,10 @@ module "role" {
 }
 
 resource "aws_transfer_user" "this" {
-  server_id = var.transfer_server
-  user_name = var.name
-  role      = module.role.iam_role_arn
-
-  # This doesn't work unless optimised directory is disabled, and that isn't available in Terraform
-  # home_directory_type = "LOGICAL"
-  # home_directory_mappings {
-  #   entry  = "/upload"
-  #   target = "/${var.landing_bucket}/${var.name}/upload"
-  # }
-
-  # home_directory_mappings {
-  #   entry  = "/download"
-  #   target = "/${var.landing_bucket}/${var.name}/download"
-  # }
-
-  # This works
-  home_directory = "/${var.landing_bucket}/${var.name}" # TODO: do we need an SFTP specific landing bucket?
+  server_id      = var.transfer_server
+  user_name      = var.name
+  role           = module.role.iam_role_arn
+  home_directory = "/${var.landing_bucket}/${var.name}"
 }
 
 resource "aws_transfer_ssh_key" "this" {

terraform/environments/analytical-platform-ingestion/observability-platform.tf

@@ -1,4 +1,6 @@
 module "observability_platform_tenant" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "ministryofjustice/observability-platform-tenant/aws"
   version = "1.0.0"
 

terraform/environments/analytical-platform-ingestion/s3-notifications.tf

@@ -1,4 +1,6 @@
 module "ingestion_landing_bucket_notification" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/s3-bucket/aws//modules/notification"
   version = "4.1.0"
 
@@ -14,6 +16,8 @@ module "ingestion_landing_bucket_notification" {
 }
 
 module "ingestion_transfer_bucket_notification" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
   source  = "terraform-aws-modules/s3-bucket/aws//modules/notification"
   version = "4.1.0"