🚀 Create analytical-platform-ingestion

terraform/environments/analytical-platform-ingestion/cloudwatch-event-rules.tf

@@ -0,0 +1,4 @@
+resource "aws_cloudwatch_event_rule" "definition_update" {
+  name                = "definition-update"
+  schedule_expression = "cron(15 6 * * ? *)" # 06:15 every day
+}

terraform/environments/analytical-platform-ingestion/cloudwatch-event-targets.tf

@@ -0,0 +1,5 @@
+resource "aws_cloudwatch_event_target" "definition_update" {
+  rule      = aws_cloudwatch_event_rule.definition_update.name
+  target_id = "definition-update"
+  arn       = module.definition_upload_lambda.lambda_function_arn
+}

terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf

@@ -0,0 +1,4 @@
+# Deactivated for now
+# resource "aws_cloudwatch_log_group" "transfer_structured_logs" {
+#   name = "/aws/transfer-structured-logs"
+# }

terraform/environments/analytical-platform-ingestion/eips.tf

@@ -0,0 +1,12 @@
+resource "aws_eip" "transfer_server" {
+  count = length(data.aws_availability_zones.available.names)
+
+  domain = "vpc"
+
+  tags = merge(
+    local.tags,
+    {
+      Name = "${local.application_name}-${local.environment}-transfer-server"
+    }
+  )
+}

terraform/environments/analytical-platform-ingestion/environment-configuration.tf

@@ -11,6 +11,16 @@ locals {
 
       /* Observability Platform */
       observability_platform = "development"
+
+      /* Image Versions */
+      scan_image_version = "0.0.4"
+      transfer_image_version = "0.0.1"
+
+      /* Target Buckets */
+      target_buckets = ["dev-ingestion-testing"]
+
+      /* Transfer Server */
+      transfer_server_hostname = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
     }
     production = {
       /* VPC */
@@ -22,6 +32,16 @@ locals {
 
       /* Observability Platform */
       observability_platform = "production"
+
+      /* Image Versions */
+      scan_image_version = "0.0.4"
+      transfer_image_version = "0.0.1"
+
+      /* Target Buckets */
+      target_buckets = ["dev-ingestion-testing"]
+
+      /* Transfer Server */
+      transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
     }
   }
 }

terraform/environments/analytical-platform-ingestion/kms-keys.tf

@@ -0,0 +1,83 @@
+module "s3_landing_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["s3/landing"]
+  description           = "Family SFTP Server, Landing S3 KMS Key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "s3_processed_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["s3/processed"]
+  description           = "Family SFTP Server, Processed S3 KMS Key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "s3_quarantine_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["s3/quarantine"]
+  description           = "Family SFTP Server, Quarantine S3 KMS Key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "s3_definitions_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["s3/definitions"]
+  description           = "Ingestion Scanning ClamAV S3 KMS Key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "sns_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["sns/notify"]
+  description           = "Key for SNS notifications"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "govuk_notify_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["secretsmanager/govuk-notify"]
+  description           = "Key for GOV.UK Notify data"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
+
+module "supplier_data_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  source  = "terraform-aws-modules/kms/aws"
+  version = "2.2.1"
+
+  aliases               = ["secretsmanager/supplier-data"]
+  description           = "Key for SFTP supplier data"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}

terraform/environments/analytical-platform-ingestion/lambda-functions.tf

@@ -0,0 +1,213 @@
+module "definition_upload_lambda" {
+  #checkov:skip=CKV_TF_1:Module is from Terraform registry
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "7.2.1"
+
+  publish        = true
+  create_package = false
+
+  function_name = "definition-upload"
+  description   = "Uploads ClamAV definitions to S3 bucket"
+  package_type  = "Image"
+  memory_size   = 2048
+  timeout       = 900
+  image_uri     = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/analytical-platform-ingestion-scan:${local.environment_configuration.scan_image_version}"
+
+  environment_variables = {
+    MODE                         = "definition-upload",
+    CLAMAV_DEFINITON_BUCKET_NAME = module.definitions_bucket.s3_bucket_id
+  }
+
+  attach_policy_statements = true
+  policy_statements = {
+    kms_access = {
+      sid    = "AllowKMS"
+      effect = "Allow"
+      actions = [
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:Encrypt",
+        "kms:DescribeKey",
+        "kms:Decrypt"
+      ]
+      resources = [module.s3_definitions_kms.key_arn]
+    },
+    s3_access = {
+      sid    = "AllowS3"
+      effect = "Allow"
+      actions = [
+        "s3:GetObject",
+        "s3:PutObject"
+      ]
+      resources = ["arn:aws:s3:::${module.definitions_bucket.s3_bucket_id}/*"]
+    }
+  }
+
+  allowed_triggers = {
+    "eventbridge" = {
+      principal  = "events.amazonaws.com"
+      source_arn = aws_cloudwatch_event_rule.definition_update.arn
+    }
+  }
+}
+
+module "scan_lambda" {
+  #checkov:skip=CKV_TF_1:Module is from Terraform registry
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "7.2.1"
+
+  publish        = true
+  create_package = false
+
+  function_name          = "scan"
+  description            = "Uses ClamAV to scan files"
+  package_type           = "Image"
+  memory_size            = 2048
+  ephemeral_storage_size = 10240
+  timeout                = 900
+  image_uri              = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/analytical-platform-ingestion-scan:${local.environment_configuration.scan_image_version}"
+
+  environment_variables = {
+    MODE                         = "scan",
+    CLAMAV_DEFINITON_BUCKET_NAME = module.definitions_bucket.s3_bucket_id
+    LANDING_BUCKET_NAME          = module.landing_bucket.s3_bucket_id
+    QUARANTINE_BUCKET_NAME       = module.quarantine_bucket.s3_bucket_id
+    PROCESSED_BUCKET_NAME        = module.processed_bucket.s3_bucket_id
+  }
+
+  attach_policy_statements = true
+  policy_statements = {
+    kms_access = {
+      sid    = "AllowKMS"
+      effect = "Allow"
+      actions = [
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:Encrypt",
+        "kms:DescribeKey",
+        "kms:Decrypt"
+      ]
+      resources = [
+        module.s3_definitions_kms.key_arn,
+        module.s3_landing_kms.key_arn,
+        module.s3_quarantine_kms.key_arn,
+        module.s3_processed_kms.key_arn,
+      ]
+    },
+    s3_access = {
+      sid    = "AllowS3"
+      effect = "Allow"
+      actions = [
+        "s3:GetObject",
+        "s3:CopyObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:PutObjectTagging"
+      ]
+      resources = [
+        "arn:aws:s3:::${module.definitions_bucket.s3_bucket_id}/*",
+        "arn:aws:s3:::${module.landing_bucket.s3_bucket_id}/*",
+        "arn:aws:s3:::${module.quarantine_bucket.s3_bucket_id}/*",
+        "arn:aws:s3:::${module.processed_bucket.s3_bucket_id}/*"
+      ]
+    }
+  }
+
+  allowed_triggers = {
+    "s3" = {
+      principal  = "s3.amazonaws.com"
+      source_arn = module.landing_bucket.s3_bucket_arn
+    }
+  }
+}
+
+module "transfer_lambda" {
+  #checkov:skip=CKV_TF_1:Module is from Terraform registry
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "7.2.1"
+
+  publish        = true
+  create_package = false
+
+  function_name          = "transfer"
+  description            = ""
+  package_type           = "Image"
+  memory_size            = 2048
+  ephemeral_storage_size = 10240
+  timeout                = 900
+  image_uri              = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/analytical-platform-ingestion-transfer:${local.environment_configuration.transfer_image_version}"
+
+  environment_variables = {
+    PROCESSED_BUCKET_NAME = module.processed_bucket.s3_bucket_id
+  }
+
+  # TODO: Check if KMS key is actually needed below
+  attach_policy_statements = true
+  policy_statements = {
+    kms_access = {
+      sid    = "AllowKMS"
+      effect = "Allow"
+      actions = [
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:Encrypt",
+        "kms:DescribeKey",
+        "kms:Decrypt"
+      ]
+      resources = [
+        module.s3_processed_kms.key_arn,
+        module.supplier_data_kms.key_arn
+      ]
+    },
+    secretsmanager_access = {
+      sid       = "AllowSecretsManager"
+      effect    = "Allow"
+      actions   = ["secretsmanager:GetSecretValue"]
+      resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:ingestion/*"]
+    },
+    s3_source_object = {
+      sid    = "AllowSourceObject"
+      effect = "Allow"
+      actions = [
+        "s3:GetObject",
+        "s3:DeleteObject",
+        "s3:GetObjectTagging"
+      ],
+      resources = ["arn:aws:s3:::${module.processed_bucket.s3_bucket_id}/*"]
+    },
+    s3_source_bucket = {
+      sid    = "AllowSourceBucket"
+      effect = "Allow"
+      actions = [
+        "s3:ListBucket"
+      ],
+      resources = ["arn:aws:s3:::${module.processed_bucket.s3_bucket_id}"]
+    },
+    s3_destination_object = {
+      sid    = "AllowDestinationObject"
+      effect = "Allow"
+      actions = [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:PutObjectTagging"
+      ]
+      resources = formatlist("arn:aws:s3:::%s/*", local.environment_configuration.target_buckets)
+    },
+    s3_destination_bucket = {
+      sid    = "AllowDestinationBucket"
+      effect = "Allow"
+      actions = [
+        "s3:ListBucket"
+      ]
+      resources = formatlist("arn:aws:s3:::%s", local.environment_configuration.target_buckets)
+    }
+  }
+
+  allowed_triggers = {
+    "s3" = {
+      principal  = "s3.amazonaws.com"
+      source_arn = module.processed_bucket.s3_bucket_arn
+    }
+  }
+}