diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d793ba473fe..13d449ef3bb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,7 +14,7 @@ /terraform/environments/cica-data-extraction @ministryofjustice/cica-extract-tool-admins @ministryofjustice/modernisation-platform /terraform/environments/cica-tariff @ministryofjustice/cica-mp-tariff @ministryofjustice/modernisation-platform /terraform/environments/contract-work-administration @ministryofjustice/laa-aws-infrastructure @ministryofjustice/laa-cwa-developer @ministryofjustice/modernisation-platform -/terraform/environments/cooker @ministryofjustice/MoJRedTeam @ministryofjustice/modernisation-platform @ministryofjustice/modernisation-platform +/terraform/environments/cooker @ministryofjustice/modernisation-platform @ministryofjustice/tvm-purple-team @ministryofjustice/modernisation-platform /terraform/environments/corporate-information-system @ministryofjustice/laa-aws-infrastructure @ministryofjustice/laa-cis-dbas @ministryofjustice/laa-cis-team @ministryofjustice/modernisation-platform /terraform/environments/corporate-staff-rostering @ministryofjustice/csr-application-support @ministryofjustice/hosting-migrations @ministryofjustice/studio-webops @ministryofjustice/modernisation-platform /terraform/environments/dacp @ministryofjustice/dts-legacy @ministryofjustice/modernisation-platform diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 66dd557d16b..3f66f070e2a 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -81,7 +81,7 @@ jobs: fetch-depth: 0 - name: Run Checkov action id: checkov - uses: bridgecrewio/checkov-action@4ad414b100f8415d05d88b6be40d7aa7aa38c057 # v12.2941.0 + uses: bridgecrewio/checkov-action@d8d05796d44fbdd67576bd6ce831ac7b3bd072bd # v12.2942.0 with: directory: ./ framework: terraform diff --git a/.github/workflows/generate-dependabot-file.yml b/.github/workflows/generate-dependabot-file.yml index 24bc2998bfc..473b3053bf3 100644 --- a/.github/workflows/generate-dependabot-file.yml +++ b/.github/workflows/generate-dependabot-file.yml @@ -16,12 +16,11 @@ defaults: shell: bash permissions: - contents: read + contents: write + pull-requests: write jobs: - create-and-commit-dependabot-file: - permissions: - pull-requests: write + create-and-commit-dependabot-file: runs-on: ubuntu-latest steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 diff --git a/scripts/generate-dependabot-file.sh b/scripts/generate-dependabot-file.sh index c69950e7b5b..3ba123fd2cf 100755 --- a/scripts/generate-dependabot-file.sh +++ b/scripts/generate-dependabot-file.sh @@ -4,6 +4,9 @@ set -euo pipefail dependabot_file=.github/dependabot.yml +# Clear the dependabot file +> $dependabot_file + # Get a list of Terraform folders all_tf_folders=`find . -type f -name '*.tf' | sed 's#/[^/]*$##' | sed 's/.\///'| sort | uniq` echo @@ -14,7 +17,7 @@ echo "Writing dependabot.yml file" # Creates a dependabot file to avoid having to manually add each new TF folder # Add any additional fixed entries in this top section cat > $dependabot_file << EOL -# This file is auto-generated here, do not manually amend. +# This file is auto-generated here, do not manually amend. # scripts/generate-dependabot.sh version: 2 @@ -24,22 +27,13 @@ updates: directory: "/" schedule: interval: "daily" - - package-ecosystem: "devcontainers" - directory: "/" - schedule: - interval: "daily" - reviewers: - - "ministryofjustice/devcontainer-community" - # Dependabot doesn't currently support wildcard or multiple directory declarations within - # a dependabot configuration, so we need to add all directories individually - # See: github.com/dependabot/dependabot-core/issues/2178 EOL -for folder in $all_tf_folders -do -echo "Generating entry for ${folder}" +echo "Generating entry for Terraform ecosystem" echo " - package-ecosystem: \"terraform\"" >> $dependabot_file -echo " directory: \"/${folder}\"" >> $dependabot_file -echo " schedule:" >> $dependabot_file -echo " interval: \"daily\"" >> $dependabot_file +echo " directories:" >> $dependabot_file +for folder in $all_tf_folders; do + echo " - \"/$folder\"" >> $dependabot_file done +echo " schedule:" >> $dependabot_file +echo " interval: \"daily\"" >> $dependabot_file \ No newline at end of file diff --git a/terraform/environments/analytical-platform-compute/cloudwatch-log-groups.tf b/terraform/environments/analytical-platform-compute/cloudwatch-log-groups.tf index 7b72c421439..7df68788bff 100644 --- a/terraform/environments/analytical-platform-compute/cloudwatch-log-groups.tf +++ b/terraform/environments/analytical-platform-compute/cloudwatch-log-groups.tf @@ -3,7 +3,7 @@ module "eks_log_group" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" - version = "5.6.1" + version = "5.7.0" name = local.eks_cloudwatch_log_group_name kms_key_id = module.eks_cluster_logs_kms.key_arn @@ -17,7 +17,7 @@ module "managed_prometheus_log_group" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" - version = "5.6.1" + version = "5.7.0" name = local.amp_cloudwatch_log_group_name kms_key_id = module.managed_prometheus_logs_kms.key_arn diff --git a/terraform/environments/analytical-platform-compute/ec2-instances.tf b/terraform/environments/analytical-platform-compute/ec2-instances.tf deleted file mode 100644 index 4104af0a722..00000000000 --- a/terraform/environments/analytical-platform-compute/ec2-instances.tf +++ /dev/null @@ -1,35 +0,0 @@ -module "debug_instance" { - #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions - - source = "terraform-aws-modules/ec2-instance/aws" - version = "5.7.1" - - name = "network-debug" - ami = "ami-0e8d228ad90af673b" # Ubuntu Server 24.04 LTS - instance_type = "t3.micro" - subnet_id = element(module.vpc.private_subnets, 0) - vpc_security_group_ids = [module.debug_instance_security_group.security_group_id] - associate_public_ip_address = false - - root_block_device = [ - { - encrypted = true - volume_type = "gp3" - volume_size = 8 - } - ] - - create_iam_instance_profile = true - iam_role_policies = { - SSMCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - - metadata_options = { - http_endpoint = "enabled" - http_put_response_hop_limit = 1 - http_tokens = "required" - instance_metadata_tags = "enabled" - } - - tags = local.tags -} diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index d5e7be40b72..4002185eb1c 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -6,7 +6,7 @@ module "eks" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/eks/aws" - version = "20.29.0" + version = "20.31.6" cluster_name = local.eks_cluster_name cluster_version = local.environment_configuration.eks_cluster_version @@ -172,7 +172,7 @@ module "karpenter" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "20.29.0" + version = "20.31.6" cluster_name = module.eks.cluster_name diff --git a/terraform/environments/analytical-platform-compute/eks-pod-identities.tf b/terraform/environments/analytical-platform-compute/eks-pod-identities.tf index 8b219126c30..ddb4efc92c4 100644 --- a/terraform/environments/analytical-platform-compute/eks-pod-identities.tf +++ b/terraform/environments/analytical-platform-compute/eks-pod-identities.tf @@ -7,7 +7,7 @@ module "aws_cloudwatch_metrics_pod_identity" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/eks-pod-identity/aws" - version = "1.7.0" + version = "1.9.0" name = "aws-cloudwatch-metrics" diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf index 6009114ee9a..47ceee3ba07 100644 --- a/terraform/environments/analytical-platform-compute/environment-configuration.tf +++ b/terraform/environments/analytical-platform-compute/environment-configuration.tf @@ -21,15 +21,15 @@ locals { /* EKS */ eks_sso_access_role = "modernisation-platform-sandbox" eks_cluster_version = "1.31" - eks_node_version = "1.26.2-360b7a38" + eks_node_version = "1.29.0-c55d099c" eks_cluster_addon_versions = { - coredns = "v1.11.3-eksbuild.2" - kube_proxy = "v1.31.2-eksbuild.2" - aws_ebs_csi_driver = "v1.36.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.9-eksbuild.1" - aws_guardduty_agent = "v1.7.1-eksbuild.2" - eks_pod_identity_agent = "v1.3.2-eksbuild.2" - vpc_cni = "v1.19.0-eksbuild.1" + coredns = "v1.11.4-eksbuild.2" + kube_proxy = "v1.31.3-eksbuild.2" + aws_ebs_csi_driver = "v1.38.1-eksbuild.1" + aws_efs_csi_driver = "v2.1.3-eksbuild.1" + aws_guardduty_agent = "v1.8.1-eksbuild.2" + eks_pod_identity_agent = "v1.3.4-eksbuild.1" + vpc_cni = "v1.19.2-eksbuild.1" } /* Data Engineering Airflow */ @@ -66,15 +66,15 @@ locals { /* EKS */ eks_sso_access_role = "modernisation-platform-developer" eks_cluster_version = "1.31" - eks_node_version = "1.26.2-360b7a38" + eks_node_version = "1.29.0-c55d099c" eks_cluster_addon_versions = { - coredns = "v1.11.3-eksbuild.2" - kube_proxy = "v1.31.2-eksbuild.2" - aws_ebs_csi_driver = "v1.36.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.9-eksbuild.1" - aws_guardduty_agent = "v1.7.1-eksbuild.2" - eks_pod_identity_agent = "v1.3.2-eksbuild.2" - vpc_cni = "v1.19.0-eksbuild.1" + coredns = "v1.11.4-eksbuild.2" + kube_proxy = "v1.31.3-eksbuild.2" + aws_ebs_csi_driver = "v1.38.1-eksbuild.1" + aws_efs_csi_driver = "v2.1.3-eksbuild.1" + aws_guardduty_agent = "v1.8.1-eksbuild.2" + eks_pod_identity_agent = "v1.3.4-eksbuild.1" + vpc_cni = "v1.19.2-eksbuild.1" } /* Data Engineering Airflow */ @@ -110,15 +110,15 @@ locals { /* EKS */ eks_sso_access_role = "modernisation-platform-developer" eks_cluster_version = "1.31" - eks_node_version = "1.26.2-360b7a38" + eks_node_version = "1.29.0-c55d099c" eks_cluster_addon_versions = { - coredns = "v1.11.3-eksbuild.2" - kube_proxy = "v1.31.2-eksbuild.2" - aws_ebs_csi_driver = "v1.36.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.9-eksbuild.1" - aws_guardduty_agent = "v1.7.1-eksbuild.2" - eks_pod_identity_agent = "v1.3.2-eksbuild.2" - vpc_cni = "v1.19.0-eksbuild.1" + coredns = "v1.11.4-eksbuild.2" + kube_proxy = "v1.31.3-eksbuild.2" + aws_ebs_csi_driver = "v1.38.1-eksbuild.1" + aws_efs_csi_driver = "v2.1.3-eksbuild.1" + aws_guardduty_agent = "v1.8.1-eksbuild.2" + eks_pod_identity_agent = "v1.3.4-eksbuild.1" + vpc_cni = "v1.19.2-eksbuild.1" } /* Data Engineering Airflow */ diff --git a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf index e37b997235b..a46f470120b 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf @@ -6,7 +6,7 @@ resource "helm_release" "actions_runner_mojas_airflow" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-airflow" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -32,7 +32,7 @@ resource "helm_release" "actions_runner_mojas_airflow_create_a_pipeline" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-airflow-create-a-pipeline" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -57,7 +57,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -81,7 +81,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_non_spot" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table-non-spot" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -109,7 +109,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_dpr" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table-dpr" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -133,7 +133,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_dpr_pp" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table-dpr-pp" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -157,7 +157,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_emds_test" /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table-emds-test" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ @@ -181,7 +181,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_emds" { /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ name = "actions-runner-mojas-create-a-derived-table-emds" repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" - version = "2.320.0-4" + version = "2.321.0" chart = "actions-runner" namespace = kubernetes_namespace.actions_runners[0].metadata[0].name values = [ diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 546726a9de2..51f3c8e75bf 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -4,7 +4,7 @@ resource "helm_release" "kyverno" { name = "kyverno" repository = "https://kyverno.github.io/kyverno" chart = "kyverno" - version = "3.3.3" + version = "3.3.4" namespace = kubernetes_namespace.kyverno.metadata[0].name values = [ templatefile( @@ -71,7 +71,7 @@ resource "helm_release" "amazon_prometheus_proxy" { name = "amazon-prometheus-proxy" repository = "https://prometheus-community.github.io/helm-charts" chart = "kube-prometheus-stack" - version = "66.2.1" + version = "67.8.0" namespace = kubernetes_namespace.aws_observability.metadata[0].name values = [ templatefile( @@ -96,7 +96,7 @@ resource "helm_release" "cluster_autoscaler" { name = "cluster-autoscaler" repository = "https://kubernetes.github.io/autoscaler" chart = "cluster-autoscaler" - version = "9.43.2" + version = "9.45.0" namespace = kubernetes_namespace.cluster_autoscaler.metadata[0].name values = [ @@ -119,7 +119,7 @@ resource "helm_release" "karpenter_crd" { name = "karpenter-crd" repository = "oci://public.ecr.aws/karpenter" chart = "karpenter-crd" - version = "1.0.8" + version = "1.1.1" namespace = kubernetes_namespace.karpenter.metadata[0].name values = [ @@ -141,7 +141,7 @@ resource "helm_release" "karpenter" { name = "karpenter" repository = "oci://public.ecr.aws/karpenter" chart = "karpenter" - version = "1.0.8" + version = "1.1.1" namespace = kubernetes_namespace.karpenter.metadata[0].name values = [ @@ -209,7 +209,7 @@ resource "helm_release" "cert_manager" { name = "cert-manager" repository = "https://charts.jetstack.io" chart = "cert-manager" - version = "v1.16.1" + version = "v1.16.2" namespace = kubernetes_namespace.cert_manager.metadata[0].name values = [ templatefile( @@ -262,7 +262,7 @@ resource "helm_release" "ingress_nginx" { name = "ingress-nginx" repository = "https://kubernetes.github.io/ingress-nginx" chart = "ingress-nginx" - version = "4.11.3" + version = "4.12.0" namespace = kubernetes_namespace.ingress_nginx.metadata[0].name values = [ templatefile( @@ -283,7 +283,7 @@ resource "helm_release" "external_secrets" { name = "external-secrets" repository = "https://charts.external-secrets.io" chart = "external-secrets" - version = "0.10.5" + version = "0.12.1" namespace = kubernetes_namespace.external_secrets.metadata[0].name values = [ templatefile( @@ -310,7 +310,7 @@ resource "helm_release" "keda" { name = "keda" repository = "https://kedacore.github.io/charts" chart = "keda" - version = "2.16.0" + version = "2.16.1" namespace = kubernetes_namespace.keda.metadata[0].name values = [ templatefile( diff --git a/terraform/environments/analytical-platform-compute/iam-policies.tf b/terraform/environments/analytical-platform-compute/iam-policies.tf index 607480dde57..9370d3e9957 100644 --- a/terraform/environments/analytical-platform-compute/iam-policies.tf +++ b/terraform/environments/analytical-platform-compute/iam-policies.tf @@ -18,7 +18,7 @@ module "eks_cluster_logs_kms_access_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "eks-cluster-logs-kms-access" @@ -45,7 +45,7 @@ module "karpenter_sqs_kms_access_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "karpenter-sqs-kms-access" @@ -71,7 +71,7 @@ module "amazon_prometheus_proxy_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "amazon-prometheus-proxy" @@ -98,7 +98,7 @@ module "managed_prometheus_kms_access_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "managed-prometheus-kms-access" @@ -147,7 +147,7 @@ module "mlflow_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "mlflow" @@ -168,7 +168,7 @@ module "gha_mojas_airflow_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "github-actions-mojas-airflow" @@ -258,7 +258,7 @@ module "analytical_platform_lake_formation_share_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "analytical-platform-lake-formation-sharing-policy" @@ -292,7 +292,7 @@ module "quicksight_vpc_connection_iam_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "quicksight-vpc-connection" @@ -341,7 +341,7 @@ module "data_production_mojap_derived_bucket_lake_formation_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "analytical-platform-data-bucket-lake-formation-policy" @@ -446,7 +446,7 @@ module "copy_apdp_cadet_metadata_to_compute_policy" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.48.0" + version = "5.52.1" name_prefix = "copy-apdp-cadet-metadata-to-compute-" diff --git a/terraform/environments/analytical-platform-compute/iam-roles.tf b/terraform/environments/analytical-platform-compute/iam-roles.tf index a27fc18735e..fa5769a590c 100644 --- a/terraform/environments/analytical-platform-compute/iam-roles.tf +++ b/terraform/environments/analytical-platform-compute/iam-roles.tf @@ -3,7 +3,7 @@ module "vpc_cni_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "vpc-cni" attach_vpc_cni_policy = true @@ -24,7 +24,7 @@ module "ebs_csi_driver_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "ebs-csi-driver" attach_ebs_csi_policy = true @@ -44,7 +44,7 @@ module "efs_csi_driver_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "efs-csi-driver" attach_efs_csi_policy = true @@ -64,7 +64,7 @@ module "aws_for_fluent_bit_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "aws-for-fluent-bit" @@ -88,7 +88,7 @@ module "amazon_prometheus_proxy_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "amazon-prometheus-proxy" @@ -111,7 +111,7 @@ module "cluster_autoscaler_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "cluster-autoscaler" @@ -133,7 +133,7 @@ module "external_dns_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "external-dns" attach_external_dns_policy = true @@ -154,7 +154,7 @@ module "cert_manager_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "cert-manager" attach_cert_manager_policy = true @@ -175,7 +175,7 @@ module "external_secrets_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "external-secrets" attach_external_secrets_policy = true @@ -196,7 +196,7 @@ module "mlflow_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" role_name_prefix = "mlflow" @@ -219,7 +219,7 @@ module "gha_mojas_airflow_iam_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-github-oidc-role" - version = "5.48.0" + version = "5.52.1" name = "github-actions-mojas-airflow" @@ -237,7 +237,7 @@ module "lake_formation_share_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" create_role = true role_requires_mfa = false @@ -264,7 +264,7 @@ module "analytical_platform_ui_service_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.48.0" + version = "5.52.1" create_role = true @@ -287,7 +287,7 @@ module "analytical_platform_control_panel_service_role" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" allow_self_assume_role = true trusted_role_arns = [ @@ -310,7 +310,7 @@ module "analytical_platform_data_eng_dba_service_role" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" allow_self_assume_role = false trusted_role_arns = formatlist("arn:aws:iam::%s:root", [local.environment_management.account_ids[local.analytical_platform_environment], local.environment_management.account_ids["analytical-platform-management-production"]]) @@ -330,7 +330,7 @@ module "quicksight_vpc_connection_iam_role" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" create_role = true role_name_prefix = "quicksight-vpc-connection" @@ -348,7 +348,7 @@ module "lake_formation_to_data_production_mojap_derived_tables_role" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" create_role = true role_requires_mfa = false @@ -378,7 +378,7 @@ module "copy_apdp_cadet_metadata_to_compute_assumable_role" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "5.48.0" + version = "5.52.1" allow_self_assume_role = false trusted_role_arns = [ diff --git a/terraform/environments/analytical-platform-compute/locals.tf b/terraform/environments/analytical-platform-compute/locals.tf index 78e3b560296..169a6155552 100644 --- a/terraform/environments/analytical-platform-compute/locals.tf +++ b/terraform/environments/analytical-platform-compute/locals.tf @@ -17,7 +17,7 @@ locals { eks_cloudwatch_log_group_retention_in_days = 400 /* Kube Prometheus Stack */ - prometheus_operator_crd_version = "v0.78.1" + prometheus_operator_crd_version = "v0.79.2" /* Mapping Analytical Platform Environments to Modernisation Platform */ @@ -31,5 +31,4 @@ locals { ) /* Environment Configuration */ environment_configuration = local.environment_configurations[local.environment] - } diff --git a/terraform/environments/analytical-platform-compute/s3-buckets.tf b/terraform/environments/analytical-platform-compute/s3-buckets.tf index 03712ea8813..f19a7bcc407 100644 --- a/terraform/environments/analytical-platform-compute/s3-buckets.tf +++ b/terraform/environments/analytical-platform-compute/s3-buckets.tf @@ -3,7 +3,7 @@ module "mlflow_bucket" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/s3-bucket/aws" - version = "4.2.2" + version = "4.3.0" bucket = "mojap-compute-${local.environment}-mlflow" @@ -46,7 +46,7 @@ module "mojap_compute_logs_bucket_eu_west_2" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/s3-bucket/aws" - version = "4.2.2" + version = "4.3.0" bucket = "mojap-compute-${local.environment}-logs-eu-west-2" @@ -101,7 +101,7 @@ module "mojap_compute_logs_bucket_eu_west_1" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/s3-bucket/aws" - version = "4.2.2" + version = "4.3.0" providers = { aws = aws.analytical-platform-compute-eu-west-1 @@ -164,7 +164,7 @@ module "mojap_compute_athena_query_results_bucket_eu_west_2" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/s3-bucket/aws" - version = "4.2.2" + version = "4.3.0" bucket = "mojap-compute-${local.environment}-athena-query-results-eu-west-2" diff --git a/terraform/environments/analytical-platform-compute/security-groups.tf b/terraform/environments/analytical-platform-compute/security-groups.tf index eb52972deeb..4812bcc7da0 100644 --- a/terraform/environments/analytical-platform-compute/security-groups.tf +++ b/terraform/environments/analytical-platform-compute/security-groups.tf @@ -55,19 +55,3 @@ module "quicksight_shared_vpc_security_group" { tags = local.tags } - -/* This security group is temporary and will be retired when we're satisfied with DataSync end-to-end */ -module "debug_instance_security_group" { - #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions - - source = "terraform-aws-modules/security-group/aws" - version = "5.2.0" - - name = "debug-instance" - vpc_id = module.vpc.vpc_id - - egress_cidr_blocks = ["0.0.0.0/0"] - egress_rules = ["all-all"] - - tags = local.tags -} diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml index 649c63ed2ae..2a7fe4a28e0 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml @@ -3,4 +3,4 @@ apiVersion: v2 name: karpenter-configuration description: A Helm chart to deploy Karpenter's configuration type: application -version: 2.3.0 +version: 3.1.0 diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml index bfaafdb48a3..882be619d42 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: bottlerocket-general diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-gpu.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-gpu.yaml index be59088d0df..947b9e40c2f 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-gpu.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-gpu.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: bottlerocket-gpu diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-airflow-high-memory.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-airflow-high-memory.yaml index bdf49b77d92..ebdb41aacfb 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-airflow-high-memory.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-airflow-high-memory.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: airflow-high-memory @@ -13,7 +13,7 @@ spec: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "airflow-high-memory" spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket-general taints: diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml index f9401e55efc..845ee26a3ad 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: general-on-demand @@ -13,7 +13,7 @@ spec: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "general-on-demand" spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket-general taints: diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml index 792f049909a..49b5caa791f 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: general-spot @@ -13,7 +13,7 @@ spec: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "general-spot" spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket-general taints: diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-on-demand.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-on-demand.yaml index 98cd1594723..da727c99e95 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-on-demand.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-on-demand.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: gpu-on-demand @@ -13,7 +13,7 @@ spec: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "gpu-on-demand" spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket-gpu taints: diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-spot.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-spot.yaml index fcefdfeb057..62b2cdd6a8b 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-spot.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-gpu-spot.yaml @@ -1,5 +1,5 @@ --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: gpu-spot @@ -13,7 +13,7 @@ spec: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "gpu-spot" spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket-gpu taints: diff --git a/terraform/environments/analytical-platform-compute/vpc-endpoints.tf b/terraform/environments/analytical-platform-compute/vpc-endpoints.tf index e096613bece..1580ccb3973 100644 --- a/terraform/environments/analytical-platform-compute/vpc-endpoints.tf +++ b/terraform/environments/analytical-platform-compute/vpc-endpoints.tf @@ -3,7 +3,7 @@ module "vpc_endpoints" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "5.15.0" + version = "5.17.0" vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets diff --git a/terraform/environments/analytical-platform-compute/vpc.tf b/terraform/environments/analytical-platform-compute/vpc.tf index 5a7d3df1099..dc3a9673094 100644 --- a/terraform/environments/analytical-platform-compute/vpc.tf +++ b/terraform/environments/analytical-platform-compute/vpc.tf @@ -6,7 +6,7 @@ module "vpc" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/vpc/aws" - version = "5.15.0" + version = "5.17.0" name = local.our_vpc_name azs = slice(data.aws_availability_zones.available.names, 0, 3) diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf new file mode 100644 index 00000000000..df37bd80250 --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf @@ -0,0 +1,30 @@ +data "aws_iam_policy_document" "datasync_cloudwatch_logs" { + statement { + sid = "AllowDataSync" + effect = "Allow" + actions = [ + "logs:PutLogEvents", + "logs:CreateLogStream" + ] + principals { + type = "Service" + identifiers = ["datasync.amazonaws.com"] + } + resources = ["${module.datasync_task_logs.cloudwatch_log_group_arn}*"] + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:aws:datasync:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:task/*"] + } + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + } +} + +resource "aws_cloudwatch_log_resource_policy" "datasync_cloudwatch_logs" { + policy_name = "datasync-cloudwatch-logs" + policy_document = data.aws_iam_policy_document.datasync_cloudwatch_logs.json +} diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf index e8bbad22e2a..849a53e325d 100644 --- a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf @@ -18,3 +18,13 @@ module "connected_vpc_route53_resolver_logs" { name = "/aws/route53-resolver/connected-vpc" retention_in_days = 400 } + +module "datasync_task_logs" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + + source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" + version = "5.6.0" + + name = "/aws/datasync/tasks" + retention_in_days = 400 +} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index bf7eff7a03b..12d32ce466e 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -1,18 +1,6 @@ -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/" - - user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] - password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] - - agent_arns = [aws_datasync_agent.main.arn] - - tags = local.tags -} - -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/" +resource "aws_datasync_location_s3" "opg_investigations" { + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn + subdirectory = "/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn @@ -21,9 +9,9 @@ resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_managemen tags = local.tags } -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/" +resource "aws_datasync_location_smb" "opg_investigations" { + server_hostname = "eucw4171nas002.dom1.infra.int" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] @@ -33,13 +21,3 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme tags = local.tags } -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/itas/" - - s3_config { - bucket_access_role_arn = module.datasync_iam_role.iam_role_arn - } - - tags = local.tags -} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf new file mode 100644 index 00000000000..7858d943ac7 --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -0,0 +1,32 @@ +resource "aws_datasync_task" "opg_investigations" { + name = "opg-investigations" + source_location_arn = aws_datasync_location_smb.opg_investigations.arn + destination_location_arn = aws_datasync_location_s3.opg_investigations.arn + cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn + + options { + gid = "NONE" + uid = "NONE" + posix_permissions = "NONE" + log_level = "TRANSFER" + verify_mode = "ONLY_FILES_TRANSFERRED" + } + + task_report_config { + report_overrides {} + report_level = "ERRORS_ONLY" + output_type = "STANDARD" + s3_object_versioning = "INCLUDE" + + s3_destination { + bucket_access_role_arn = module.datasync_iam_role.iam_role_arn + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn + } + } + + # schedule { + # schedule_expression = "cron(0 23 ? * THU *)" + # } + + tags = local.tags +} diff --git a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf index 516cc0f9fa7..faa96ea33c4 100644 --- a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf +++ b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf @@ -20,9 +20,9 @@ locals { ] /* Image Versions */ - scan_image_version = "0.1.3" - transfer_image_version = "0.0.18" - notify_image_version = "0.0.19" + scan_image_version = "0.1.4" + transfer_image_version = "0.0.19" + notify_image_version = "0.0.20" /* Target Buckets */ target_buckets = ["mojap-land-dev"] @@ -65,9 +65,9 @@ locals { ] /* Image Versions */ - scan_image_version = "0.1.3" - transfer_image_version = "0.0.18" - notify_image_version = "0.0.19" + scan_image_version = "0.1.4" + transfer_image_version = "0.0.19" + notify_image_version = "0.0.20" /* Target Buckets */ target_buckets = ["mojap-land", "mojap-ingestion-${local.environment}-ext-2024-target"] diff --git a/terraform/environments/analytical-platform-ingestion/iam-policies.tf b/terraform/environments/analytical-platform-ingestion/iam-policies.tf index 885365cb44a..b7f5c588806 100644 --- a/terraform/environments/analytical-platform-ingestion/iam-policies.tf +++ b/terraform/environments/analytical-platform-ingestion/iam-policies.tf @@ -35,7 +35,7 @@ data "aws_iam_policy_document" "datasync" { "kms:DescribeKey", "kms:Decrypt", ] - resources = [module.transfer_logs_kms.key_arn] + resources = [module.s3_datasync_kms.key_arn] } statement { sid = "AllowS3BucketActions" @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "datasync" { "s3:ListBucket", "s3:ListBucketMultipartUploads" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "AllowS3ObjectActions" @@ -61,7 +61,7 @@ data "aws_iam_policy_document" "datasync" { "s3:PutObject", "s3:PutObjectTagging" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } @@ -116,7 +116,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetReplicationConfiguration", "s3:ListBucket" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "SourceBucketObjectPermissions" @@ -127,7 +127,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetObjectVersionTagging", "s3:ObjectOwnerOverrideToBucketOwner" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } diff --git a/terraform/environments/analytical-platform-ingestion/s3.tf b/terraform/environments/analytical-platform-ingestion/s3.tf index bfeff2602fb..c4cab8508b8 100644 --- a/terraform/environments/analytical-platform-ingestion/s3.tf +++ b/terraform/environments/analytical-platform-ingestion/s3.tf @@ -161,57 +161,56 @@ module "bold_egress_bucket" { } } - -module "datasync_bucket" { +module "datasync_opg_investigations_bucket" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions source = "terraform-aws-modules/s3-bucket/aws" version = "4.1.2" - bucket = "mojap-ingestion-${local.environment}-datasync" + bucket = "mojap-ingestion-${local.environment}-datasync-opg-investigations" force_destroy = true - versioning = { - enabled = true - } - - replication_configuration = { - role = module.datasync_replication_iam_role.iam_role_arn - rules = [ - { - id = "datasync-replication" - status = "Enabled" - delete_marker_replication = true - - source_selection_criteria = { - sse_kms_encrypted_objects = { - enabled = true - } - } - - destination = { - account_id = local.environment_management.account_ids["analytical-platform-data-production"] - bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" - storage_class = "STANDARD" - access_control_translation = { - owner = "Destination" - } - encryption_configuration = { - replica_kms_key_id = local.environment_configuration.mojap_land_kms_key - } - metrics = { - status = "Enabled" - minutes = 15 - } - replication_time = { - status = "Enabled" - minutes = 15 - } - } - } - ] - } + # versioning = { + # enabled = true + # } + + # replication_configuration = { + # role = module.datasync_replication_iam_role.iam_role_arn + # rules = [ + # { + # id = "datasync-replication" + # status = "Enabled" + # delete_marker_replication = true + + # source_selection_criteria = { + # sse_kms_encrypted_objects = { + # enabled = true + # } + # } + + # destination = { + # account_id = local.environment_management.account_ids["analytical-platform-data-production"] + # bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" + # storage_class = "STANDARD" + # access_control_translation = { + # owner = "Destination" + # } + # encryption_configuration = { + # replica_kms_key_id = local.environment_configuration.mojap_land_kms_key + # } + # metrics = { + # status = "Enabled" + # minutes = 15 + # } + # replication_time = { + # status = "Enabled" + # minutes = 15 + # } + # } + # } + # ] + # } server_side_encryption_configuration = { rule = { diff --git a/terraform/environments/analytical-platform-ingestion/security-groups.tf b/terraform/environments/analytical-platform-ingestion/security-groups.tf index a3539ac2d2e..04b400e6df9 100644 --- a/terraform/environments/analytical-platform-ingestion/security-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/security-groups.tf @@ -90,7 +90,7 @@ module "datasync_activation_nlb_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = ["${local.environment_configuration.datasync_instance_private_ip}/32"] - egress_rules = ["http-80-tcp", ] + egress_rules = ["http-80-tcp"] ingress_cidr_blocks = ["${data.external.external_ip.result["ip"]}/32"] ingress_rules = ["http-80-tcp"] @@ -110,7 +110,7 @@ module "datasync_vpc_endpoint_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = [module.connected_vpc.vpc_cidr_block] - egress_rules = ["all-all", ] + egress_rules = ["all-all"] ingress_with_cidr_blocks = [ { @@ -160,6 +160,16 @@ module "datasync_instance_security_group" { vpc_id = module.connected_vpc.vpc_id + egress_with_cidr_blocks = [ + { + from_port = 445 + to_port = 445 + protocol = "tcp" + description = "SMB" + cidr_blocks = "10.0.0.0/8" + } + ] + egress_with_source_security_group_id = [ { from_port = 1024 diff --git a/terraform/environments/ccms-ebs/application_variables.json b/terraform/environments/ccms-ebs/application_variables.json index 6cbad9e4871..2bbe65077bb 100644 --- a/terraform/environments/ccms-ebs/application_variables.json +++ b/terraform/environments/ccms-ebs/application_variables.json @@ -169,7 +169,7 @@ "ebs_size_ebsdb_exhome": 100, "ebs_size_ebsdb_u01": 300, "ebs_size_ebsdb_arch": 500, - "ebs_iops_ebsdb_dbf01": 7000, + "ebs_iops_ebsdb_dbf01": 24000, "ebs_size_ebsdb_dbf01": 5500, "ebs_iops_ebsdb_dbf02": 14000, "ebs_size_ebsdb_dbf02": 5500, diff --git a/terraform/environments/corporate-staff-rostering/locals_preproduction.tf b/terraform/environments/corporate-staff-rostering/locals_preproduction.tf index 909a02e7131..c1180ab1b58 100644 --- a/terraform/environments/corporate-staff-rostering/locals_preproduction.tf +++ b/terraform/environments/corporate-staff-rostering/locals_preproduction.tf @@ -708,10 +708,7 @@ locals { route53_zones = { "pp.csr.service.justice.gov.uk" = { records = [ - # Set to IP of the Azure CSR PP DB in PPCDL00019 - { name = "ppiwfm", type = "A", ttl = "300", records = ["10.40.42.132"] }, - { name = "ppiwfm-a", type = "A", ttl = "300", records = ["10.40.42.132"] }, - { name = "ppiwfm-b", type = "CNAME", ttl = "300", records = ["pp-csr-db-a.corporate-staff-rostering.hmpps-preproduction.modernisation-platform.service.justice.gov.uk"] }, + { name = "ppiwfm", type = "CNAME", ttl = "300", records = ["pp-csr-db-a.corporate-staff-rostering.hmpps-preproduction.modernisation-platform.service.justice.gov.uk"] }, ] lb_alias_records = [ { name = "r1", type = "A", lbs_map_key = "r12" }, diff --git a/terraform/environments/corporate-staff-rostering/locals_production.tf b/terraform/environments/corporate-staff-rostering/locals_production.tf index b50c3ccbd69..f24d73cb75b 100644 --- a/terraform/environments/corporate-staff-rostering/locals_production.tf +++ b/terraform/environments/corporate-staff-rostering/locals_production.tf @@ -648,7 +648,7 @@ locals { records = [ { name = "test", type = "NS", ttl = "86400", records = ["ns-1332.awsdns-38.org", "ns-2038.awsdns-62.co.uk", "ns-62.awsdns-07.com", "ns-689.awsdns-22.net"] }, { name = "pp", type = "NS", ttl = "86400", records = ["ns-1408.awsdns-48.org", "ns-1844.awsdns-38.co.uk", "ns-447.awsdns-55.com", "ns-542.awsdns-03.net"] }, - { name = "piwfm", type = "A", ttl = "300", records = ["10.40.8.132"] }, + { name = "piwfm", type = "CNAME", ttl = "300", records = ["pd-csr-db-a.corporate-staff-rostering.hmpps-production.modernisation-platform.service.justice.gov.uk"] }, { name = "traina", type = "CNAME", ttl = "300", records = ["traina.pp.csr.service.justice.gov.uk"] }, { name = "trainb", type = "CNAME", ttl = "300", records = ["trainb.pp.csr.service.justice.gov.uk"] }, ] diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json index 2763ff66608..dd994ca64ec 100644 --- a/terraform/environments/digital-prison-reporting/application_variables.json +++ b/terraform/environments/digital-prison-reporting/application_variables.json @@ -101,6 +101,8 @@ "dps-inc-reporting", "dps-csip", "dps-alerts", + "dps-use-of-force", + "dps-locations", "dps-testing" ], "alarms": { @@ -181,7 +183,7 @@ "enable_dbt_k8s_secrets": true, "dpr_generic_athena_workgroup": true, "analytics_generic_athena_workgroup": true, - "redshift_table_expiry_seconds": "604800", + "redshift_table_expiry_days": 7, "enable_s3_data_migrate_lambda": true }, "test": { @@ -284,7 +286,9 @@ "dps-basm", "dps-inc-reporting", "dps-csip", - "dps-alerts" + "dps-alerts", + "dps-use-of-force", + "dps-locations" ], "alarms": { "setup_cw_alarms": true, @@ -364,7 +368,7 @@ "enable_dbt_k8s_secrets": true, "dpr_generic_athena_workgroup": true, "analytics_generic_athena_workgroup": true, - "redshift_table_expiry_seconds": "604800", + "redshift_table_expiry_days": 7, "enable_s3_data_migrate_lambda": true }, "preproduction": { @@ -469,7 +473,9 @@ "dps-basm", "dps-inc-reporting", "dps-csip", - "dps-alerts" + "dps-alerts", + "dps-use-of-force", + "dps-locations" ], "alarms": { "setup_cw_alarms": true, @@ -567,7 +573,7 @@ ] } ], - "redshift_table_expiry_seconds": "604800", + "redshift_table_expiry_days": 7, "enable_s3_data_migrate_lambda": true }, "production": { @@ -670,7 +676,9 @@ "dps-basm", "dps-inc-reporting", "dps-csip", - "dps-alerts" + "dps-alerts", + "dps-use-of-force", + "dps-locations" ], "alarms": { "setup_cw_alarms": true, @@ -765,7 +773,7 @@ ] } ], - "redshift_table_expiry_seconds": "86400", + "redshift_table_expiry_days": 1, "enable_s3_data_migrate_lambda": false } } diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index f3bd7fe74f2..2552462ca76 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -240,10 +240,12 @@ locals { lambda_redshift_table_expiry_cluster_id = module.datamart.cluster_id lambda_redshift_table_expiry_database_name = module.datamart.cluster_database_name lambda_redshift_table_expiry_schedule_expression = "rate(1 hour)" - lambda_redshift_table_expiry_seconds = local.application_data.accounts[local.environment].redshift_table_expiry_seconds + lambda_redshift_table_expiry_seconds = (local.application_data.accounts[local.environment].redshift_table_expiry_days * 86400) lambda_redshift_table_expiry_timeout_seconds = 900 lambda_redshift_table_expiry_memory_size = 1024 + s3_redshift_table_expiry_days = local.application_data.accounts[local.environment].redshift_table_expiry_days + 1 + reporting_lambda_code_s3_key = "build-artifacts/digital-prison-reporting-lambdas/jars/digital-prison-reporting-lambdas-vLatest-all.jar" # s3 transfer diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index da4e02427af..a013155a45b 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -995,16 +995,17 @@ module "s3_working_bucket" { custom_kms_key = local.s3_kms_arn create_notification_queue = false # For SQS Queue enable_lifecycle = true - enable_lifecycle_expiration = true lifecycle_category = "long_term" override_expiration_rules = [ { - prefix = "reports" - days = 7 + id = "reports" + prefix = "reports/" + days = local.s3_redshift_table_expiry_days }, { - prefix = "dpr" + id = "dpr" + prefix = "dpr/" days = 7 } ] diff --git a/terraform/environments/digital-prison-reporting/modules/domains/ingestion-pipeline/pipeline.tf b/terraform/environments/digital-prison-reporting/modules/domains/ingestion-pipeline/pipeline.tf index 12c44bbf026..ae7bf7cdaa0 100644 --- a/terraform/environments/digital-prison-reporting/modules/domains/ingestion-pipeline/pipeline.tf +++ b/terraform/environments/digital-prison-reporting/modules/domains/ingestion-pipeline/pipeline.tf @@ -198,7 +198,6 @@ module "data_ingestion_pipeline" { "--dpr.datastorage.retry.minWaitMillis" : tostring(var.glue_s3_retry_min_wait_millis), "--dpr.datastorage.retry.maxWaitMillis" : tostring(var.glue_s3_retry_max_wait_millis), "--dpr.config.s3.bucket" : var.s3_glue_bucket_id, - "--dpr.allowed.s3.file.extensions" : ".parquet", "--dpr.config.key" : var.domain } }, diff --git a/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf b/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf index de837ca2de5..c30ea5ffc73 100644 --- a/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf +++ b/terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf @@ -253,7 +253,8 @@ module "replay_pipeline" { "JobName" : var.glue_unprocessed_raw_files_check_job, "Arguments" : { "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds), - "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts) + "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts), + "--dpr.allowed.s3.file.regex" : "\\d+-\\d+.parquet" } }, "Next" : "Empty Raw Data" diff --git a/terraform/environments/digital-prison-reporting/modules/s3_bucket/main.tf b/terraform/environments/digital-prison-reporting/modules/s3_bucket/main.tf index bbc6b8f94a0..f49d93d5b1f 100644 --- a/terraform/environments/digital-prison-reporting/modules/s3_bucket/main.tf +++ b/terraform/environments/digital-prison-reporting/modules/s3_bucket/main.tf @@ -32,6 +32,8 @@ resource "aws_s3_bucket_public_access_block" "storage" { } resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" { + #checkov:skip=CKV_AWS_300:TODO Will be addressed as part of https://dsdmoj.atlassian.net/browse/DPR2-1083 + # Create the lifecycle configuration if either lifecycle or Intelligent-Tiering is enabled count = var.enable_lifecycle || var.enable_intelligent_tiering ? 1 : 0 @@ -102,6 +104,23 @@ resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" { storage_class = "INTELLIGENT_TIERING" } } + + # Expiration rules + dynamic "rule" { + for_each = var.override_expiration_rules + content { + id = rule.value.id + status = "Enabled" + + filter { + prefix = rule.value.prefix + } + + expiration { + days = rule.value.days + } + } + } } resource "aws_s3_bucket_server_side_encryption_configuration" "encryption" { diff --git a/terraform/environments/digital-prison-reporting/modules/s3_bucket/variables.tf b/terraform/environments/digital-prison-reporting/modules/s3_bucket/variables.tf index 683e34f1677..1163851d784 100644 --- a/terraform/environments/digital-prison-reporting/modules/s3_bucket/variables.tf +++ b/terraform/environments/digital-prison-reporting/modules/s3_bucket/variables.tf @@ -1,6 +1,7 @@ variable "name" { description = "Name of the Bucket" + type = string default = "" } @@ -24,11 +25,13 @@ variable "cloudtrail_access_policy" { variable "s3_notification_name" { description = "S3 Notification Event Name" + type = string default = "s3-notification-event" } variable "create_s3" { description = "Setup S3 Buckets" + type = bool default = false } @@ -40,60 +43,46 @@ variable "custom_kms_key" { variable "create_notification_queue" { description = "Setup Notification Queue" + type = bool default = false } variable "sqs_msg_retention_seconds" { description = "SQS Message Retention" + type = number default = 86400 } variable "filter_prefix" { description = "S3 Notification Filter Prefix" + type = string default = null } variable "enable_lifecycle" { description = "Enabled Lifecycle for S3 Storage, Default is False" + type = bool default = false } -#variable "expiration_days" { -# description = "Days to wait before deleting expired items." -# default = 90 -#} - -#variable "expiration_prefix_redshift" { -# description = "Directory Prefix where Redshift Async query results are stored to apply expiration to." -# default = "/" -#} - -#variable "expiration_prefix_athena" { -# description = "Directory Prefix where Athena Async query results are stored to apply expiration to." -# default = "/" -#} - variable "enable_versioning_config" { description = "Enable Versioning Config for S3 Storage, Default is Disabled" + type = string default = "Disabled" } variable "enable_s3_versioning" { description = "Enable Versioning for S3 Bucket, Default is false" + type = bool default = false } variable "enable_notification" { description = "Enable S3 Bucket Notifications, Default is false" + type = bool default = false } -#variable "bucket_notifications" { -# type = map(any) -# description = "AWS S3 Bucket Notifications" -# default = null -#} - variable "bucket_notifications" { type = any description = "AWS S3 Bucket Notifications" @@ -112,12 +101,13 @@ variable "dependency_lambda" { variable "bucket_key" { description = "If Bucket Key is Enabled or Disabled" + type = bool default = true } ## Dynamic override_expiration_rules variable "override_expiration_rules" { - type = list(object({ prefix = string, days = number })) + type = list(object({ id = string, prefix = string, days = number })) default = [] } @@ -126,11 +116,6 @@ variable "lifecycle_category" { default = "standard" # Options: "short_term", "long_term", "temporary", "standard" } -variable "enable_lifecycle_expiration" { - description = "Enable item expiration - requires 'enable_lifecycle' and 'override_expiration_rules' to be defined/enabled." - default = false -} - variable "enable_intelligent_tiering" { description = "Enable Intelligent-Tiering storage class for S3 bucket" type = bool diff --git a/terraform/environments/electronic-monitoring-data/analytical_platform_share.tf b/terraform/environments/electronic-monitoring-data/analytical_platform_share.tf index 257019dc701..fde781851e3 100644 --- a/terraform/environments/electronic-monitoring-data/analytical_platform_share.tf +++ b/terraform/environments/electronic-monitoring-data/analytical_platform_share.tf @@ -4,8 +4,11 @@ locals { dbt_k8s_secrets_placeholder = { oidc_cluster_identifier = "placeholder2" } - admin_roles = local.is-development ? "sandbox" : "data-eng" - suffix = local.is-production ? "" : "-test" + admin_roles = local.is-development ? "sandbox" : "data-eng" + suffix = local.is-production ? "" : "-test" + prod_dbs_to_grant = local.is-production ? ["am_stg", "cap_dw_stg", "emd_historic_int", "historic_api_mart", "historic_api_mart_mock"] : [] + dev_dbs_to_grant = local.is-production ? [for db in local.prod_dbs_to_grant : "${db}_historic_dev_dbt"] : [] + dbs_to_grant = toset(flatten([local.prod_dbs_to_grant, local.dev_dbs_to_grant])) } # Source Analytics DBT Secrets @@ -414,7 +417,6 @@ resource "aws_iam_role_policy_attachment" "analytical_platform_share_policy_atta policy_arn = "arn:aws:iam::aws:policy/AWSLakeFormationCrossAccountManager" } - resource "aws_lakeformation_data_lake_settings" "lake_formation" { admins = flatten([[for share in local.analytical_platform_share : aws_iam_role.analytical_platform_share_role[share.target_account_name].arn], data.aws_iam_session_context.current.issuer_arn, try(one(data.aws_iam_roles.data_engineering_roles.arns), [])]) @@ -432,3 +434,19 @@ resource "aws_lakeformation_data_lake_settings" "lake_formation" { principal = "IAM_ALLOWED_PRINCIPALS" } } + +module "share_dbs_with_de_role" { + count = local.is-production ? 1 : 0 + source = "./modules/lakeformation_database_share" + dbs_to_grant = local.dbs_to_grant + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) +} + +module "share_dbs_with_cadt_role" { + count = local.is-production ? 1 : 0 + source = "./modules/lakeformation_database_share" + dbs_to_grant = local.dbs_to_grant + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + role_arn = aws_iam_role.dataapi_cross_role.arn +} diff --git a/terraform/environments/electronic-monitoring-data/cloud_platform_share.tf b/terraform/environments/electronic-monitoring-data/cloud_platform_share.tf index 414d6e83d31..2f8d0a5a0ef 100644 --- a/terraform/environments/electronic-monitoring-data/cloud_platform_share.tf +++ b/terraform/environments/electronic-monitoring-data/cloud_platform_share.tf @@ -33,9 +33,10 @@ module "cmt_front_end_assumable_role" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" version = "5.48.0" - trusted_role_arns = [ - local.resolved-cloud-platform-iam-role - ] + trusted_role_arns = flatten([ + local.resolved-cloud-platform-iam-role, + data.aws_iam_roles.data_engineering_roles.arns + ]) create_role = true role_requires_mfa = false @@ -45,7 +46,27 @@ module "cmt_front_end_assumable_role" { tags = local.tags } +module "specials_cmt_front_end_assumable_role" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + #checkov:skip=CKV_TF_2:Module registry does not support tags for versions + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" + version = "5.48.0" + + trusted_role_arns = flatten([ + local.resolved-cloud-platform-iam-role, + data.aws_iam_roles.data_engineering_roles.arns + ]) + + create_role = true + role_requires_mfa = false + + role_name = "specials_cmt_read_emds_data_${local.environment_shorthand}" + + tags = local.tags +} + # module "share_api_data_marts" { +# count = local.is-production ? 1 : 0 # #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions # #checkov:skip=CKV_TF_2:Module registry does not support tags for versions # source = "github.com/ministryofjustice/terraform-aws-analytical-platform-lakeformation?ref=32525da937012178e430585ac5a00f05193f58eb" @@ -67,3 +88,77 @@ module "cmt_front_end_assumable_role" { # aws.destination = aws # } # } + + +data "aws_iam_policy_document" "standard_athena_access" { + statement { + actions = [ + "athena:GetDataCatalog", + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:GetWorkGroup", + "athena:StartQueryExecution", + "athena:StopQueryExecution" + ] + resources = [ + "arn:aws:athena:${data.aws_region.current.name}:${local.env_account_id}:*/*" + ] + } + statement { + actions = [ + "athena:ListWorkGroups" + ] + resources = [ + "*" + ] + } + statement { + actions = ["lakeformation:GetDataAccess"] + resources = ["*"] + } + statement { + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts" + ] + resources = [module.s3-athena-bucket.bucket.arn] + } + statement { + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts" + ] + resources = ["${module.s3-athena-bucket.bucket.arn}/*"] + } + statement { + effect = "Allow" + actions = [ + "glue:GetDatabases", + "glue:GetDatabase", + "glue:GetTables", + "glue:GetTable" + ] + resources = [ + "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:catalog", + "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:database/staged_fms_${local.env_}dbt", + "arn:aws:glue:${data.aws_region.current.name}:${local.env_account_id}:table/staged_fms_${local.env_}dbt/*" + ] + } +} + +resource "aws_iam_policy" "standard_athena_access" { + name_prefix = "standard_athena_access" + description = "Standard permissions for Athena" + policy = data.aws_iam_policy_document.standard_athena_access.json +} + +resource "aws_iam_role_policy_attachment" "standard_athena_access" { + policy_arn = aws_iam_policy.standard_athena_access.arn + role = module.cmt_front_end_assumable_role.iam_role_name +} diff --git a/terraform/environments/electronic-monitoring-data/dms_data_validation_glue_job_v2.tf b/terraform/environments/electronic-monitoring-data/dms_data_validation_glue_job_v2.tf index 559b87ded21..8d931646ff7 100644 --- a/terraform/environments/electronic-monitoring-data/dms_data_validation_glue_job_v2.tf +++ b/terraform/environments/electronic-monitoring-data/dms_data_validation_glue_job_v2.tf @@ -559,7 +559,12 @@ resource "aws_glue_job" "etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm" { role_arn = aws_iam_role.glue_mig_and_val_iam_role.arn glue_version = "4.0" worker_type = "G.2X" - number_of_workers = 4 + number_of_workers = 2 + + execution_property { + max_concurrent_runs = 12 + } + default_arguments = { "--script_bucket_name" = module.s3-glue-job-script-bucket.bucket.id "--rds_db_host_ep" = split(":", aws_db_instance.database_2022.endpoint)[0] @@ -578,6 +583,7 @@ resource "aws_glue_job" "etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm" { "--rds_db_table_hashed_rows_parent_dir" = "rds_tables_rows_hashed" "--incremental_run_bool" = "false" "--rds_query_where_clause" = "" + "--df_where_clause" = "" "--skip_columns_for_hashing" = "" "--coalesce_int" = 0 "--extra-py-files" = "s3://${module.s3-glue-job-script-bucket.bucket.id}/${aws_s3_object.aws_s3_object_pyzipfile_to_s3folder.id}" diff --git a/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py b/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py index 7e4e9c193f1..a55ff60f38d 100644 --- a/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py +++ b/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py @@ -61,6 +61,7 @@ OPTIONAL_INPUTS = [ "rds_query_where_clause", + "df_where_clause", "coalesce_int", "parallel_jdbc_conn_num", "pkey_lower_bound_int", @@ -371,6 +372,10 @@ def write_rds_df_to_s3_parquet(df_rds_write: DataFrame, "month", F.month(date_partition_column_name)) # ---------------------------------------------------------- + df_where_clause = args.get('df_where_clause', None) + if df_where_clause is not None: + rds_hashed_rows_df = rds_hashed_rows_df.where(f"{df_where_clause}") + if rds_yyyy_mm_df_repartition_num != 0: # Note: Default 'partitionby_columns' values may not be appropriate for all the scenarios. # So, the user can edit the list-'partitionby_columns' value(s) if required at runtime. diff --git a/terraform/environments/electronic-monitoring-data/lake_formation_cloud_platform.tf b/terraform/environments/electronic-monitoring-data/lake_formation_cloud_platform.tf new file mode 100644 index 00000000000..06edefa2b13 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/lake_formation_cloud_platform.tf @@ -0,0 +1,69 @@ +locals { + env_ = "${local.environment_shorthand}_" + cap_dw_tables = local.is-production ? ["contact_history", "equipment_details", "event_history", "incident", "order_details", "services", "suspension_of_visits", "violations", "visit_details"] : [] + am_tables = local.is-production ? ["am_contact_history", "am_equipment_details", "am_incident", "am_order_details", "am_services", "am_visit_details"] : [] +} + +resource "aws_lakeformation_resource" "data_bucket" { + arn = module.s3-create-a-derived-table-bucket.bucket.arn +} + +module "share_current_version" { + count = local.is-test ? 1 : 0 + source = "./modules/lakeformation_w_data_filter" + table_filters = { + "account" = "__current=true" + } + role_arn = module.cmt_front_end_assumable_role.iam_role_arn + database_name = "staged_fms_${local.env_}dbt" + data_engineer_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn +} + +# module "cap_dw_excluding_specials" { +# for_each = toset(local.cap_dw_tables) +# source = "./modules/lakeformation_w_data_filter" +# table_filters = { +# (each.key) = "specials_flag=0" +# } +# role_arn = module.cmt_front_end_assumable_role.iam_role_arn +# database_name = "historic_api_mart_tables_historic_dev_dbt" +# data_engineer_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) +# data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn +# } + +# module "cap_dw_including_specials" { +# for_each = toset(local.cap_dw_tables) +# source = "./modules/lakeformation_w_data_filter" +# table_filters = { +# (each.key) = "" +# } +# role_arn = module.specials_cmt_front_end_assumable_role.iam_role_arn +# database_name = "historic_api_mart_tables_historic_dev_dbt" +# data_engineer_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) +# data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn +# } + +# module "am_for_non_specials_role" { +# for_each = toset(local.am_tables) +# source = "./modules/lakeformation_w_data_filter" +# table_filters = { +# (each.key) = "" +# } +# role_arn = module.cmt_front_end_assumable_role.iam_role_arn +# database_name = "historic_api_mart_tables_historic_dev_dbt" +# data_engineer_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) +# data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn +# } + +# module "am_for_specials_role" { +# for_each = toset(local.am_tables) +# source = "./modules/lakeformation_w_data_filter" +# table_filters = { +# (each.key) = "" +# } +# role_arn = module.specials_cmt_front_end_assumable_role.iam_role_arn +# database_name = "historic_api_mart_tables_historic_dev_dbt" +# data_engineer_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) +# data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn +# } diff --git a/terraform/environments/electronic-monitoring-data/lambda_triggers.tf b/terraform/environments/electronic-monitoring-data/lambda_triggers.tf index 73ae3a3a3f8..f773523c9a1 100644 --- a/terraform/environments/electronic-monitoring-data/lambda_triggers.tf +++ b/terraform/environments/electronic-monitoring-data/lambda_triggers.tf @@ -1,34 +1,59 @@ # --------------------------------------- # live fms data json trigger # --------------------------------------- -resource "aws_sns_topic_subscription" "live_serco_fms_sns_subscription" { - topic_arn = aws_sns_topic.live_serco_fms_s3_events.arn - protocol = "lambda" - endpoint = module.format_json_fms_data.lambda_function_arn +resource "aws_s3_bucket_notification" "historic_data_store" { + depends_on = [aws_lambda_permission.historic, aws_lambda_permission.live_serco_fms] + bucket = module.s3-data-bucket.bucket.id + + lambda_function { + lambda_function_arn = module.calculate_checksum.lambda_function_arn + events = [ + "s3:ObjectCreated:*" + ] + filter_suffix = ".bak" + } + lambda_function { + lambda_function_arn = module.calculate_checksum.lambda_function_arn + events = [ + "s3:ObjectCreated:*", + ] + filter_suffix = ".zip" + } + lambda_function { + lambda_function_arn = module.calculate_checksum.lambda_function_arn + events = [ + "s3:ObjectCreated:*", + ] + filter_suffix = ".bacpac" + } + lambda_function { + lambda_function_arn = module.format_json_fms_data.lambda_function_arn + events = [ + "s3:ObjectCreated:*", + ] + filter_suffix = ".JSON" + filter_prefix = "serco/fms/" + } } -resource "aws_lambda_permission" "live_serco_fms_with_sns" { - statement_id = "LiveServcoFMSLambdaAllowExecutionFromSNS" + +resource "aws_lambda_permission" "live_serco_fms" { + statement_id = "LiveSercoFMSLambdaAllowExecutionFromS3Bucket" action = "lambda:InvokeFunction" function_name = module.format_json_fms_data.lambda_function_name - principal = "sns.amazonaws.com" - source_arn = aws_sns_topic.live_serco_fms_s3_events.arn + principal = "s3.amazonaws.com" + source_arn = module.s3-data-bucket.bucket.arn } # --------------------------------------- # historic data json trigger # --------------------------------------- -resource "aws_sns_topic_subscription" "historic_sns_subscription" { - topic_arn = aws_sns_topic.historic_s3_events.arn - protocol = "lambda" - endpoint = module.calculate_checksum.lambda_function_arn -} -resource "aws_lambda_permission" "historic_with_sns" { - statement_id = "ChecksumLambdaAllowExecutionFromHistoricDataSNS" +resource "aws_lambda_permission" "historic" { + statement_id = "ChecksumLambdaAllowExecutionFromHistoricData" action = "lambda:InvokeFunction" function_name = module.calculate_checksum.lambda_function_name - principal = "sns.amazonaws.com" - source_arn = aws_sns_topic.historic_s3_events.arn + principal = "s3.amazonaws.com" + source_arn = module.s3-data-bucket.bucket.arn } diff --git a/terraform/environments/electronic-monitoring-data/lambdas_main.tf b/terraform/environments/electronic-monitoring-data/lambdas_main.tf index 1d50436299b..9c5cd6b463e 100644 --- a/terraform/environments/electronic-monitoring-data/lambdas_main.tf +++ b/terraform/environments/electronic-monitoring-data/lambdas_main.tf @@ -89,17 +89,17 @@ module "rotate_iam_key" { #----------------------------------------------------------------------------------- module "virus_scan_definition_upload" { - source = "./modules/lambdas" - function_name = "definition-upload" - is_image = true - ecr_repo_name = "analytical-platform-ingestion-scan" - function_tag = "0.1.0" - role_name = aws_iam_role.virus_scan_definition_upload.name - role_arn = aws_iam_role.virus_scan_definition_upload.arn - memory_size = 2048 - timeout = 900 - security_group_ids = [aws_security_group.lambda_generic.id] - subnet_ids = data.aws_subnets.shared-public.ids + source = "./modules/lambdas" + function_name = "definition-upload" + is_image = true + ecr_repo_name = "analytical-platform-ingestion-scan" + function_tag = "0.1.3" + role_name = aws_iam_role.virus_scan_definition_upload.name + role_arn = aws_iam_role.virus_scan_definition_upload.arn + memory_size = 2048 + timeout = 900 + # security_group_ids = [aws_security_group.lambda_generic.id] + # subnet_ids = data.aws_subnets.shared-public.ids core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"] environment_variables = { MODE = "definition-upload", @@ -124,7 +124,7 @@ module "virus_scan_file" { function_name = "scan" is_image = true ecr_repo_name = "analytical-platform-ingestion-scan" - function_tag = "0.1.0" + function_tag = "0.1.3" role_name = aws_iam_role.virus_scan_file.name role_arn = aws_iam_role.virus_scan_file.arn ephemeral_storage_size = 10240 diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/main.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/main.tf new file mode 100644 index 00000000000..4421754a16b --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/main.tf @@ -0,0 +1,31 @@ +data "aws_caller_identity" "current" {} + +resource "aws_lakeformation_permissions" "s3_bucket_permissions" { + principal = var.role_arn + + permissions = ["DATA_LOCATION_ACCESS"] + + data_location { + arn = var.data_bucket_lf_resource + } +} + + +resource "aws_lakeformation_permissions" "grant_cadt_databases" { + for_each = var.dbs_to_grant + principal = var.role_arn + permissions = ["ALL"] + database { + name = each.value + } +} + +resource "aws_lakeformation_permissions" "grant_cadt_tables" { + for_each = var.dbs_to_grant + principal = var.role_arn + permissions = ["ALL"] + table { + database_name = each.value + wildcard = true + } +} diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/variables.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/variables.tf new file mode 100644 index 00000000000..4b0fec38c70 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/variables.tf @@ -0,0 +1,15 @@ +variable "dbs_to_grant" { + description = "Name of the database the table belongs to" + type = set(string) + +} + +variable "data_bucket_lf_resource" { + description = "arn of the lake formation resource for the data bucket" + type = string +} + +variable "role_arn" { + description = "Role to grant permissions to" + type = string +} diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/versions.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/versions.tf new file mode 100644 index 00000000000..2b58ee107cb --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_database_share/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + required_version = ">= 1.0.1" +} diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/main.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/main.tf new file mode 100644 index 00000000000..7ea3e917022 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/main.tf @@ -0,0 +1,82 @@ +data "aws_caller_identity" "current" {} + +resource "aws_lakeformation_permissions" "data_engineering_permissions" { + permissions = ["ALL"] + principal = var.data_engineer_role_arn + + database { + name = var.database_name + } +} + +resource "random_id" "suffix" { + byte_length = 32 +} + +resource "aws_lakeformation_permissions" "data_engineering_table_permissions" { + for_each = var.table_filters + permissions = ["ALL"] + principal = var.data_engineer_role_arn + + table { + database_name = var.database_name + name = each.key + } +} + +resource "aws_lakeformation_permissions" "de_s3_bucket_permissions" { + principal = var.data_engineer_role_arn + + permissions = ["DATA_LOCATION_ACCESS"] + + data_location { + arn = var.data_bucket_lf_resource + } +} + +resource "aws_lakeformation_permissions" "s3_bucket_permissions" { + principal = var.role_arn + + permissions = ["DATA_LOCATION_ACCESS"] + + data_location { + arn = var.data_bucket_lf_resource + } +} + +resource "aws_lakeformation_data_cells_filter" "data_filter" { + for_each = tomap(var.table_filters) + table_data { + database_name = var.database_name + name = "filter-${each.key}-${random_id.suffix.hex}" + table_catalog_id = data.aws_caller_identity.current.account_id + table_name = each.key + column_wildcard { + excluded_column_names = [] + } + dynamic "row_filter" { + for_each = each.value != "" ? [each.value] : [] + content { + filter_expression = each.value + } + } + dynamic "row_filter" { + for_each = each.value == "" ? [each.value] : [] + content { + all_rows_wildcard {} + } + } + } +} + +resource "aws_lakeformation_permissions" "share_filtered_data_with_role" { + for_each = tomap(var.table_filters) + principal = var.role_arn + permissions = ["DESCRIBE", "SELECT"] + data_cells_filter { + database_name = var.database_name + table_name = each.key + table_catalog_id = data.aws_caller_identity.current.account_id + name = aws_lakeformation_data_cells_filter.data_filter[each.key].table_data[0].name + } +} diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/variables.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/variables.tf new file mode 100644 index 00000000000..1f940fb7971 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/variables.tf @@ -0,0 +1,25 @@ +variable "table_filters" { + description = "Map of names of the tables and filters to apply" + type = map(string) +} + +variable "database_name" { + description = "Name of the database the table belongs to" + type = string + +} + +variable "data_engineer_role_arn" { + description = "ARN of the DE role" + type = string +} + +variable "data_bucket_lf_resource" { + description = "arn of the lake formation resource for the data bucket" + type = string +} + +variable "role_arn" { + description = "Role to grant permissions to" + type = string +} diff --git a/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/versions.tf b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/versions.tf new file mode 100644 index 00000000000..2b58ee107cb --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/lakeformation_w_data_filter/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + required_version = ">= 1.0.1" +} diff --git a/terraform/environments/electronic-monitoring-data/s3_sns.tf b/terraform/environments/electronic-monitoring-data/s3_sns.tf deleted file mode 100644 index b821c1d8722..00000000000 --- a/terraform/environments/electronic-monitoring-data/s3_sns.tf +++ /dev/null @@ -1,119 +0,0 @@ - -# bucket notification for data store -resource "aws_s3_bucket_notification" "historic_data_store" { - depends_on = [aws_sns_topic_policy.historic_s3_events_policy] - bucket = module.s3-data-bucket.bucket.id - - # Only for copy events as those are events triggered by data being copied - #  from landing bucket. - topic { - topic_arn = aws_sns_topic.historic_s3_events.arn - events = [ - "s3:ObjectCreated:*" - ] - filter_suffix = ".bak" - } - topic { - topic_arn = aws_sns_topic.historic_s3_events.arn - events = [ - "s3:ObjectCreated:*", - ] - filter_suffix = ".zip" - } - topic { - topic_arn = aws_sns_topic.historic_s3_events.arn - events = [ - "s3:ObjectCreated:*", - ] - filter_suffix = ".bacpac" - } -} - -# sns topic to allow multiple lambdas to be triggered off of it -#trivy:ignore:AVD-AWS-0136 -resource "aws_sns_topic" "historic_s3_events" { - name = "${module.s3-data-bucket.bucket.id}-historic-object-created-topic" - kms_master_key_id = "alias/aws/sns" -} - -# IAM policy document for the SNS topic policy -data "aws_iam_policy_document" "historic_sns_policy" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["s3.amazonaws.com"] - } - - actions = ["SNS:Publish"] - resources = [aws_sns_topic.historic_s3_events.arn] - - condition { - test = "ArnLike" - variable = "aws:SourceArn" - values = [module.s3-data-bucket.bucket.arn] - } - } -} - -# Apply policy to the SNS topic -resource "aws_sns_topic_policy" "historic_s3_events_policy" { - arn = aws_sns_topic.historic_s3_events.arn - policy = data.aws_iam_policy_document.historic_sns_policy.json -} - -# ----------------------------------------------- -# Live data sns notification -# ----------------------------------------------- - - -# bucket notification for data store -resource "aws_s3_bucket_notification" "live_serco_fms_data_store" { - depends_on = [aws_sns_topic_policy.live_serco_fms_s3_events_policy] - bucket = module.s3-data-bucket.bucket.id - - # Only for copy events as those are events triggered by data being copied - #  from landing bucket. - topic { - topic_arn = aws_sns_topic.live_serco_fms_s3_events.arn - events = [ - "s3:ObjectCreated:*" - ] - filter_suffix = ".JSON" - } -} - -# sns topic to allow multiple lambdas to be triggered off of it -#trivy:ignore:AVD-AWS-0136 -resource "aws_sns_topic" "live_serco_fms_s3_events" { - name = "${module.s3-data-bucket.bucket.id}-live-object-created-topic" - kms_master_key_id = "alias/aws/sns" -} - -# IAM policy document for the SNS topic policy -data "aws_iam_policy_document" "live_serco_fms_sns_policy" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["s3.amazonaws.com"] - } - - actions = ["SNS:Publish"] - resources = [aws_sns_topic.live_serco_fms_s3_events.arn] - - condition { - test = "ArnLike" - variable = "aws:SourceArn" - values = [module.s3-data-bucket.bucket.arn] - } - } -} - -# Apply policy to the SNS topic -resource "aws_sns_topic_policy" "live_serco_fms_s3_events_policy" { - arn = aws_sns_topic.live_serco_fms_s3_events.arn - policy = data.aws_iam_policy_document.live_serco_fms_sns_policy.json -} diff --git a/terraform/environments/example/data.tf b/terraform/environments/example/data.tf index 67a57156522..c77538f5fca 100644 --- a/terraform/environments/example/data.tf +++ b/terraform/environments/example/data.tf @@ -3,16 +3,3 @@ ########################################################################################### #### This file can be used to store data specific to the member account #### - -#For macie code -# data "aws_s3_bucket" "bucket1" { -# bucket = module.bastion_linux.bastion_s3_bucket.bucket.id -# } - -# data "aws_s3_bucket" "bucket2" { -# bucket = "config-20220505080423816000000003" -# } - -# data "aws_s3_bucket" "bucket3" { -# bucket = module.s3-bucket.bucket.id -# } \ No newline at end of file diff --git a/terraform/environments/example/macie.tf b/terraform/environments/example/macie.tf deleted file mode 100644 index a67034b7b67..00000000000 --- a/terraform/environments/example/macie.tf +++ /dev/null @@ -1,29 +0,0 @@ -# ########################################################################################## -# # ------------------------Comment out file if not required---------------------------------- -# ########################################################################################## - -# # Create macie account - -# resource "aws_macie2_account" "example" { -# finding_publishing_frequency = "ONE_HOUR" -# status = "ENABLED" -# } - -# # Now create a job - -# resource "aws_macie2_classification_job" "example" { -# job_type = "ONE_TIME" -# name = "" -# s3_job_definition { -# bucket_definitions { -# account_id = local.environment_management.account_ids[terraform.workspace] -# buckets = [ -# data.aws_s3_bucket.bucket1.id, -# data.aws_s3_bucket.bucket2.id, -# data.aws_s3_bucket.bucket3.id, -# ] -# } -# } -# job_status = "RUNNING" -# depends_on = [aws_macie2_account.example] -# } \ No newline at end of file diff --git a/terraform/environments/nomis/locals_cloudwatch_metric_alarms.tf b/terraform/environments/nomis/locals_cloudwatch_metric_alarms.tf index ae742fa4155..c4d72ceeafb 100644 --- a/terraform/environments/nomis/locals_cloudwatch_metric_alarms.tf +++ b/terraform/environments/nomis/locals_cloudwatch_metric_alarms.tf @@ -75,6 +75,15 @@ locals { module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux, module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status_os, module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status_app, + local.environment == "production" ? { + free-disk-space-low = merge(module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux["free-disk-space-low"], { + threshold = "90" + }) + } : { + free-disk-space-low = merge(module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux["free-disk-space-low"], { + threshold = "85" + }) + } ) xtag = merge( diff --git a/terraform/environments/oasys-national-reporting/locals_ec2_instances.tf b/terraform/environments/oasys-national-reporting/locals_ec2_instances.tf index e68f62a5ed8..cd393770985 100644 --- a/terraform/environments/oasys-national-reporting/locals_ec2_instances.tf +++ b/terraform/environments/oasys-national-reporting/locals_ec2_instances.tf @@ -50,6 +50,9 @@ locals { update-ssm-agent = "patchgroup1" } cloudwatch_metric_alarms = merge( + module.baseline_presets.cloudwatch_metric_alarms.ec2, + module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_windows, + module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_or_cwagent_stopped_windows, local.cloudwatch_metric_alarms.windows, local.cloudwatch_metric_alarms.bods, ) diff --git a/terraform/environments/oasys-national-reporting/locals_lbs.tf b/terraform/environments/oasys-national-reporting/locals_lbs.tf index d1bd4e9025a..8844cc4bea4 100644 --- a/terraform/environments/oasys-national-reporting/locals_lbs.tf +++ b/terraform/environments/oasys-national-reporting/locals_lbs.tf @@ -10,7 +10,7 @@ locals { force_destroy_bucket = true internal_lb = false load_balancer_type = "application" - security_groups = ["public-lb"] + security_groups = ["public-lb", "public-lb-2"] subnets = module.environment.subnets["public"].ids listeners = { diff --git a/terraform/environments/oasys-national-reporting/locals_preproduction.tf b/terraform/environments/oasys-national-reporting/locals_preproduction.tf index ac1ec74c4ce..eb601b1299d 100644 --- a/terraform/environments/oasys-national-reporting/locals_preproduction.tf +++ b/terraform/environments/oasys-national-reporting/locals_preproduction.tf @@ -98,7 +98,6 @@ locals { instance_type = "r6i.2xlarge" disable_api_termination = true }) - cloudwatch_metric_alarms = null tags = merge(local.ec2_instances.bods.tags, { oasys-national-reporting-environment = "pp" domain-name = "azure.hmpp.root" diff --git a/terraform/environments/oasys-national-reporting/locals_security_groups.tf b/terraform/environments/oasys-national-reporting/locals_security_groups.tf index 965602e839e..3881b00d050 100644 --- a/terraform/environments/oasys-national-reporting/locals_security_groups.tf +++ b/terraform/environments/oasys-national-reporting/locals_security_groups.tf @@ -6,8 +6,10 @@ locals { enduserclient_internal = flatten([ "10.0.0.0/8", ]) - enduserclient_public = flatten([ - module.ip_addresses.moj_cidrs.trusted_moj_digital_staff_public, + enduserclient_public1 = flatten([ + module.ip_addresses.moj_cidrs.trusted_moj_digital_staff_public + ]) + enduserclient_public2 = flatten([ module.ip_addresses.azure_fixngo_cidrs.internet_egress, module.ip_addresses.mp_cidrs.live_eu_west_nat, ]) @@ -28,8 +30,10 @@ locals { enduserclient_internal = [ "10.0.0.0/8" ] - enduserclient_public = flatten([ - module.ip_addresses.moj_cidrs.trusted_moj_digital_staff_public, + enduserclient_public1 = flatten([ + module.ip_addresses.moj_cidrs.trusted_moj_digital_staff_public + ]) + enduserclient_public2 = flatten([ module.ip_addresses.azure_fixngo_cidrs.internet_egress, module.ip_addresses.mp_cidrs.live_eu_west_nat, ]) @@ -63,19 +67,19 @@ locals { protocol = -1 self = true } - http_lb = { + http = { description = "Allow http ingress" from_port = 80 to_port = 80 protocol = "TCP" - cidr_blocks = local.security_group_cidrs.enduserclient_public + cidr_blocks = local.security_group_cidrs.enduserclient_public1 } - https_lb = { - description = "Allow enduserclient https ingress" + https = { + description = "Allow https ingress" from_port = 443 to_port = 443 protocol = "TCP" - cidr_blocks = local.security_group_cidrs.enduserclient_public + cidr_blocks = local.security_group_cidrs.enduserclient_public1 } } egress = { @@ -88,6 +92,42 @@ locals { } } } + public-lb-2 = { + description = "Security group for public load balancer part 2" + ingress = { + all-within-subnet = { + description = "Allow all ingress to self" + from_port = 0 + to_port = 0 + protocol = -1 + self = true + } + http = { + description = "Allow http ingress" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = local.security_group_cidrs.enduserclient_public2 + } + https = { + description = "Allow https ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = local.security_group_cidrs.enduserclient_public2 + } + } + egress = { + all = { + description = "Allow all egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_groups = [] + } + } + } lb = { description = "Security group for public subnet" ingress = { diff --git a/terraform/environments/oasys-national-reporting/locals_test.tf b/terraform/environments/oasys-national-reporting/locals_test.tf index b72084b85f7..328e9b5dcb3 100644 --- a/terraform/environments/oasys-national-reporting/locals_test.tf +++ b/terraform/environments/oasys-national-reporting/locals_test.tf @@ -153,6 +153,7 @@ locals { t2-onr-bods-1 = merge(local.ec2_instances.bods, { config = merge(local.ec2_instances.bods.config, { + ami_name = "hmpps_windows_server_2019_release_2024-12-02T00-00-37.662Z" availability_zone = "eu-west-2a" instance_profile_policies = concat(local.ec2_instances.bods.config.instance_profile_policies, [ "Ec2SecretPolicy", @@ -170,6 +171,7 @@ locals { t2-onr-bods-2 = merge(local.ec2_instances.bods, { config = merge(local.ec2_instances.bods.config, { + ami_name = "hmpps_windows_server_2019_release_2024-12-02T00-00-37.662Z" availability_zone = "eu-west-2b" instance_profile_policies = concat(local.ec2_instances.bods.config.instance_profile_policies, [ "Ec2SecretPolicy", diff --git a/terraform/environments/oasys/locals_production.tf b/terraform/environments/oasys/locals_production.tf index 090857a5fb8..50455aa1bd7 100644 --- a/terraform/environments/oasys/locals_production.tf +++ b/terraform/environments/oasys/locals_production.tf @@ -772,6 +772,8 @@ locals { { name = "t2-b", type = "CNAME", ttl = "3600", records = ["public-lb-1856376477.eu-west-2.elb.amazonaws.com"] }, { name = "t2-b-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] }, { name = "t2-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] }, + { name = "t2-c", type = "CNAME", ttl = "3600", records = ["public-lb-1856376477.eu-west-2.elb.amazonaws.com"] }, + { name = "t2-c-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] }, { name = "_9f1b86e95d13d2cc7b9629f67d672c40", type = "CNAME", ttl = "86400", records = ["_7ea92a123c65795698dd19834dd71f61.fdbjvjdfdx.acm-validations.aws."] }, { name = "_26aaae7b839510727c2dd323b483ea5d.pp", type = "CNAME", ttl = "86400", records = ["_72222d02a82256bb6d75c872bc7bc1aa.qxcwttcyyb.acm-validations.aws."] }, diff --git a/terraform/environments/oasys/locals_test.tf b/terraform/environments/oasys/locals_test.tf index a85cd272e15..55b5d64d0c3 100644 --- a/terraform/environments/oasys/locals_test.tf +++ b/terraform/environments/oasys/locals_test.tf @@ -91,6 +91,13 @@ locals { t2-oasys-web-b = merge(local.ec2_autoscaling_groups.web, { # For SAN project (OASYS replacement) requested by Howard Smith # Autoscaling disabled as initially server will be configured manually + autoscaling_group = merge(local.ec2_autoscaling_groups.web.autoscaling_group, { + desired_capacity = 1 # setting to 0 leaves in a stopped state because of the warm_pool config below + warm_pool = { + min_size = 0 + reuse_on_scale_in = true + } + }) config = merge(local.ec2_autoscaling_groups.web.config, { ami_name = "oasys_webserver_release_*" availability_zone = "eu-west-2b" @@ -513,3 +520,7 @@ locals { } } } + + + + diff --git a/terraform/environments/observability-platform/environment-configurations.tf b/terraform/environments/observability-platform/environment-configurations.tf index 656dd941e2d..6e2377b1405 100644 --- a/terraform/environments/observability-platform/environment-configurations.tf +++ b/terraform/environments/observability-platform/environment-configurations.tf @@ -226,6 +226,13 @@ locals { amazon_prometheus_query_enabled = false xray_enabled = false athena_enabled = false + }, + "modernisation-platform" = { + cloudwatch_enabled = true + prometheus_push_enabled = false + amazon_prometheus_query_enabled = false + xray_enabled = false + athena_enabled = false } } } diff --git a/terraform/environments/observability-platform/iam-policies.tf b/terraform/environments/observability-platform/iam-policies.tf index 20c99915579..e60a8259a71 100644 --- a/terraform/environments/observability-platform/iam-policies.tf +++ b/terraform/environments/observability-platform/iam-policies.tf @@ -4,7 +4,10 @@ data "aws_iam_policy_document" "amazon_managed_grafana_remote_cloudwatch" { effect = "Allow" actions = ["sts:AssumeRole"] resources = [ - for account in local.all_aws_accounts : format("arn:aws:iam::%s:role/observability-platform", local.environment_management.account_ids[account]) + for account in local.all_aws_accounts : format( + "arn:aws:iam::%s:role/observability-platform", + account == "modernisation-platform" ? local.environment_management.modernisation_platform_account_id : local.environment_management.account_ids[account] + ) ] } } diff --git a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf index 285c5fb6202..6f2b7c92ccf 100644 --- a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf +++ b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf @@ -17,7 +17,7 @@ module "cloudwatch_source" { source = "../../grafana/cloudwatch-source" name = each.key - account_id = var.environment_management.account_ids[each.key] + account_id = try(each.key == "modernisation-platform" ? var.environment_management.modernisation_platform_account_id : var.environment_management.account_ids[each.key], var.environment_management.account_ids[each.key]) cloudwatch_custom_namespaces = try(each.value.cloudwatch_custom_namespaces, null) xray_enabled = try(each.value.xray_enabled, false) @@ -32,7 +32,7 @@ module "amazon_prometheus_query_source" { source = "../../grafana/amazon-prometheus-query-source" name = each.key - account_id = var.environment_management.account_ids[each.key] + account_id = try(each.key == "modernisation-platform" ? var.environment_management.modernisation_platform_account_id : var.environment_management.account_ids[each.key], var.environment_management.account_ids[each.key]) amazon_prometheus_workspace_region = try(each.value.amazon_prometheus_workspace_region, "eu-west-2") amazon_prometheus_workspace_id = each.value.amazon_prometheus_workspace_id } @@ -71,7 +71,7 @@ module "prometheus_push" { source = "../../prometheus/iam-role" name = each.key - account_id = var.environment_management.account_ids[each.key] + account_id = try(each.key == "modernisation-platform" ? var.environment_management.modernisation_platform_account_id : var.environment_management.account_ids[each.key], var.environment_management.account_ids[each.key]) } module "team" { diff --git a/terraform/environments/ppud/lambda_scripts/ppud_elb_report_prod.py b/terraform/environments/ppud/lambda_scripts/ppud_elb_report_prod.py index 0e4d8aae316..2819f62b9ce 100644 --- a/terraform/environments/ppud/lambda_scripts/ppud_elb_report_prod.py +++ b/terraform/environments/ppud/lambda_scripts/ppud_elb_report_prod.py @@ -18,7 +18,7 @@ # Configuration CURRENT_DATE = datetime.now().strftime('%a %d %b %Y') SENDER = "donotreply@cjsm.secure-email.ppud.justice.gov.uk" -RECIPIENTS = ['nick.buckingham@colt.net', 'kofi-nimoh@colt.net'] +RECIPIENTS = ['nick.buckingham@colt.net', 'kofi.owusu-nimoh@colt.net'] SUBJECT = f'AWS PPUD Load Balancer Report - {CURRENT_DATE}' AWS_REGION = 'eu-west-2' ELB_NAME = "app/PPUD-ALB/9d129853721723f4" # Replace with your ELB name diff --git a/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py b/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py index 034d7385609..07941797d7c 100644 --- a/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py +++ b/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py @@ -26,7 +26,7 @@ END_TIME = datetime.utcnow() START_TIME = END_TIME - timedelta(hours=9) SENDER = "donotreply@cjsm.secure-email.ppud.justice.gov.uk" -RECIPIENTS = ["nick.buckingham@colt.net", "pankaj.pant@colt.net", "david.savage@colt.net", "kofi-nimoh@colt.net", "helen.stimpson@colt.net"] +RECIPIENTS = ["nick.buckingham@colt.net", "pankaj.pant@colt.net", "david.savage@colt.net", "kofi.owusu-nimoh@colt.net", "helen.stimpson@colt.net"] SUBJECT = f'AWS EC2 CPU Utilization Report - {SERVER} - {CURRENT_DATE}' REGION = "eu-west-2" IMAGE_ID = "ami-02f8251c8cdf2464f" diff --git a/terraform/environments/ppud/sns.tf b/terraform/environments/ppud/sns.tf index 62c908fb2f5..c0559aec03f 100644 --- a/terraform/environments/ppud/sns.tf +++ b/terraform/environments/ppud/sns.tf @@ -27,13 +27,6 @@ resource "aws_sns_topic_subscription" "cw_sms_subscription" { endpoint = "+447903642202" # Nick Buckingham } -resource "aws_sns_topic_subscription" "cw_email_subscription" { - count = local.is-production == true ? 1 : 0 - topic_arn = aws_sns_topic.cw_alerts[0].arn - protocol = "email" - endpoint = "nbuckingham@gmail.com" -} - /* resource "aws_sns_topic_subscription" "cw_sms_subscription1" { count = local.is-production == true ? 1 : 0