Merge pull request #6224 from ministryofjustice/ap-compute-continued-2

🎡 Analytical Platform Compute EKS
ministryofjustice · May 23, 2024 · e7b2952 · e7b2952
2 parents e18cfb2 + 9125f3a
commit e7b2952
Show file tree

Hide file tree

Showing 9 changed files with 130 additions and 35 deletions.
diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json
@@ -1,24 +1,24 @@
 {
   "features": {
-    "ghcr.io/ministryofjustice/devcontainer-feature/aws:0": {
-      "version": "0.0.3",
-      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/aws@sha256:249e6f164df67ce38c8217c25e52d89ac1267d79536669647eed14fd2609f715",
-      "integrity": "sha256:249e6f164df67ce38c8217c25e52d89ac1267d79536669647eed14fd2609f715"
+    "ghcr.io/ministryofjustice/devcontainer-feature/aws:1": {
+      "version": "1.0.0",
+      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/aws@sha256:bb07a76c8e7a6b630a2056ce959addddee436e3f9936c69b9163eff54f58dbd5",
+      "integrity": "sha256:bb07a76c8e7a6b630a2056ce959addddee436e3f9936c69b9163eff54f58dbd5"
     },
-    "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:0": {
-      "version": "0.0.6",
-      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes@sha256:f8fdb7e830dbd88a495ec8fe013bd4f99a512f1f83ddef9fbd116b32a0cf3bda",
-      "integrity": "sha256:f8fdb7e830dbd88a495ec8fe013bd4f99a512f1f83ddef9fbd116b32a0cf3bda"
+    "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:1": {
+      "version": "1.0.1",
+      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes@sha256:0ec758e44468ba2a8b70b87613762ab04e50f7bb5eac8f2aea592cff213dbde5",
+      "integrity": "sha256:0ec758e44468ba2a8b70b87613762ab04e50f7bb5eac8f2aea592cff213dbde5"
     },
-    "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:0": {
-      "version": "0.0.3",
-      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis@sha256:81efa45affc66c168d273817f6f86a64f90715e9482eb7f6e3b33af006a2236c",
-      "integrity": "sha256:81efa45affc66c168d273817f6f86a64f90715e9482eb7f6e3b33af006a2236c"
+    "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:1": {
+      "version": "1.0.0",
+      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis@sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16",
+      "integrity": "sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16"
     },
-    "ghcr.io/ministryofjustice/devcontainer-feature/terraform:0": {
-      "version": "0.0.5",
-      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/terraform@sha256:d9915b6c4320f13835ce7fd08887421c9dde1c38a3f17acd303a22f6fdaf5f00",
-      "integrity": "sha256:d9915b6c4320f13835ce7fd08887421c9dde1c38a3f17acd303a22f6fdaf5f00"
+    "ghcr.io/ministryofjustice/devcontainer-feature/terraform:1": {
+      "version": "1.0.0",
+      "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/terraform@sha256:af3b3891cf31ff373df29998c690257d6f21f2ee4536bc4d692856408ef0c83a",
+      "integrity": "sha256:af3b3891cf31ff373df29998c690257d6f21f2ee4536bc4d692856408ef0c83a"
     }
   }
 }
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,9 +2,9 @@
   "name": "modernisation-platform-environments",
   "image": "ghcr.io/ministryofjustice/devcontainer-base:latest",
   "features": {
-    "ghcr.io/ministryofjustice/devcontainer-feature/aws:0": {},
-    "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:0": {},
-    "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:0": {},
-    "ghcr.io/ministryofjustice/devcontainer-feature/terraform:0": {}
+    "ghcr.io/ministryofjustice/devcontainer-feature/aws:1": {},
+    "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:1": {},
+    "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:1": {},
+    "ghcr.io/ministryofjustice/devcontainer-feature/terraform:1": {}
   }
 }
diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf
@@ -6,7 +6,7 @@ module "eks" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/eks/aws"
-  version = "20.10.0"
+  version = "20.11.1"
 
   cluster_name    = local.eks_cluster_name
   cluster_version = local.environment_configuration.eks_cluster_version

diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf
@@ -32,16 +32,19 @@ locals {
       vpc_one_nat_gateway_per_az = true
       vpc_single_nat_gateway     = false
 
+      /* Route53 */
+      route53_zone = "compute.development.analytical-platform.service.justice.gov.uk"
+
       /* EKS */
       eks_sso_access_role = "modernisation-platform-sandbox"
       eks_cluster_version = "1.29"
       eks_node_version    = "1.20.0-fcf71a47"
       eks_cluster_addon_versions = {
         coredns                = "v1.11.1-eksbuild.9"
         kube_proxy             = "v1.29.3-eksbuild.2"
-        aws_ebs_csi_driver     = "v1.30.0-eksbuild.1"
+        aws_ebs_csi_driver     = "v1.31.0-eksbuild.1"
         aws_efs_csi_driver     = "v2.0.2-eksbuild.1"
-        aws_guardduty_agent    = "v1.5.0-eksbuild.1"
+        aws_guardduty_agent    = "v1.6.1-eksbuild.1"
         eks_pod_identity_agent = "v1.2.0-eksbuild.1"
         vpc_cni                = "v1.18.1-eksbuild.3"
       }
@@ -64,16 +67,19 @@ locals {
       vpc_one_nat_gateway_per_az = true
       vpc_single_nat_gateway     = false
 
+      /* Route53 */
+      route53_zone = "compute.test.analytical-platform.service.justice.gov.uk"
+
       /* EKS */
       eks_sso_access_role = "modernisation-platform-developer"
       eks_cluster_version = "1.29"
       eks_node_version    = "1.20.0-fcf71a47"
       eks_cluster_addon_versions = {
         coredns                = "v1.11.1-eksbuild.9"
         kube_proxy             = "v1.29.3-eksbuild.2"
-        aws_ebs_csi_driver     = "v1.30.0-eksbuild.1"
+        aws_ebs_csi_driver     = "v1.31.0-eksbuild.1"
         aws_efs_csi_driver     = "v2.0.2-eksbuild.1"
-        aws_guardduty_agent    = "v1.5.0-eksbuild.1"
+        aws_guardduty_agent    = "v1.6.1-eksbuild.1"
         eks_pod_identity_agent = "v1.2.0-eksbuild.1"
         vpc_cni                = "v1.18.1-eksbuild.3"
       }
@@ -96,16 +102,19 @@ locals {
       vpc_one_nat_gateway_per_az = true
       vpc_single_nat_gateway     = false
 
+      /* Route53 */
+      route53_zone = "compute.analytical-platform.service.justice.gov.uk"
+
       /* EKS */
       eks_sso_access_role = "modernisation-platform-developer"
       eks_cluster_version = "1.29"
       eks_node_version    = "1.20.0-fcf71a47"
       eks_cluster_addon_versions = {
         coredns                = "v1.11.1-eksbuild.9"
         kube_proxy             = "v1.29.3-eksbuild.2"
-        aws_ebs_csi_driver     = "v1.30.0-eksbuild.1"
+        aws_ebs_csi_driver     = "v1.31.0-eksbuild.1"
         aws_efs_csi_driver     = "v2.0.2-eksbuild.1"
-        aws_guardduty_agent    = "v1.5.0-eksbuild.1"
+        aws_guardduty_agent    = "v1.6.1-eksbuild.1"
         eks_pod_identity_agent = "v1.2.0-eksbuild.1"
         vpc_cni                = "v1.18.1-eksbuild.3"
       }

diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf
@@ -1,5 +1,6 @@
 /* Policy */
 resource "helm_release" "kyverno" {
+  /* https://artifacthub.io/packages/helm/kyverno/kyverno */
   name       = "kyverno"
   repository = "https://kyverno.github.io/kyverno"
   chart      = "kyverno"
@@ -21,6 +22,7 @@ resource "helm_release" "kyverno" {
   The Helm chart also doesn't have support for IRSA, so a EKS Pod Identity has been been made ready to use module.aws_cloudwatch_metrics_pod_identity
 */
 resource "helm_release" "aws_cloudwatch_metrics" {
+  /* https://artifacthub.io/packages/helm/aws/aws-cloudwatch-metrics */
   name       = "aws-cloudwatch-metrics"
   repository = "https://aws.github.io/eks-charts"
   chart      = "aws-cloudwatch-metrics"
@@ -39,6 +41,7 @@ resource "helm_release" "aws_cloudwatch_metrics" {
 }
 
 resource "helm_release" "aws_for_fluent_bit" {
+  /* https://artifacthub.io/packages/helm/aws/aws-for-fluent-bit */
   name       = "aws-for-fluent-bit"
   repository = "https://aws.github.io/eks-charts"
   chart      = "aws-for-fluent-bit"
@@ -60,6 +63,7 @@ resource "helm_release" "aws_for_fluent_bit" {
 }
 
 resource "helm_release" "amazon_prometheus_proxy" {
+  /* https://artifacthub.io/packages/helm/prometheus-community/prometheus */
   name       = "amazon-prometheus-proxy"
   repository = "https://prometheus-community.github.io/helm-charts"
   chart      = "prometheus"
@@ -75,4 +79,28 @@ resource "helm_release" "amazon_prometheus_proxy" {
       }
     )
   ]
+
+  depends_on = [module.amazon_prometheus_proxy_iam_role]
+}
+
+/* Cluster */
+resource "helm_release" "cluster_autoscaler" {
+  /* https://artifacthub.io/packages/helm/cluster-autoscaler/cluster-autoscaler */
+  name       = "cluster-autoscaler"
+  repository = "https://kubernetes.github.io/autoscaler"
+  chart      = "cluster-autoscaler"
+  version    = "9.37.0"
+  namespace  = "kube-system"
+
+  values = [
+    templatefile(
+      "${path.module}/src/helm/cluster-autoscaler/values.yml.tftpl",
+      {
+        aws_region   = data.aws_region.current.name
+        cluster_name = module.eks.cluster_name
+        eks_role_arn = module.cluster_autoscaler_iam_role.iam_role_arn
+      }
+    )
+  ]
+  depends_on = [module.cluster_autoscaler_iam_role]
 }
diff --git a/terraform/environments/analytical-platform-compute/iam-policies.tf b/terraform/environments/analytical-platform-compute/iam-policies.tf
@@ -18,7 +18,7 @@ module "eks_cluster_logs_kms_access_iam_policy" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.39.0"
+  version = "5.39.1"
 
   name_prefix = "eks-cluster-logs-kms-access"
 
@@ -44,7 +44,7 @@ module "amazon_prometheus_proxy_iam_policy" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.39.0"
+  version = "5.39.1"
 
   name_prefix = "amazon-prometheus-proxy"
 
@@ -71,7 +71,7 @@ module "managed_prometheus_kms_access_iam_policy" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.39.0"
+  version = "5.39.1"
 
   name_prefix = "managed-prometheus-kms-access"
 

diff --git a/terraform/environments/analytical-platform-compute/iam-roles.tf b/terraform/environments/analytical-platform-compute/iam-roles.tf
@@ -3,7 +3,7 @@ module "vpc_cni_iam_role" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.39.0"
+  version = "5.39.1"
 
   role_name_prefix      = "vpc-cni"
   attach_vpc_cni_policy = true
@@ -24,7 +24,7 @@ module "ebs_csi_driver_iam_role" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.39.0"
+  version = "5.39.1"
 
   role_name_prefix      = "ebs-csi-driver"
   attach_ebs_csi_policy = true
@@ -44,7 +44,7 @@ module "efs_csi_driver_iam_role" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.39.0"
+  version = "5.39.1"
 
   role_name_prefix      = "efs-csi-driver"
   attach_efs_csi_policy = true
@@ -64,7 +64,7 @@ module "aws_for_fluent_bit_iam_role" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.39.0"
+  version = "5.39.1"
 
   role_name_prefix = "aws-for-fluent-bit"
 
@@ -88,7 +88,7 @@ module "amazon_prometheus_proxy_iam_role" {
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.39.0"
+  version = "5.39.1"
 
   role_name_prefix = "amazon-prometheus-proxy"
 
@@ -105,3 +105,25 @@ module "amazon_prometheus_proxy_iam_role" {
 
   tags = local.tags
 }
+
+module "cluster_autoscaler_iam_role" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "5.39.1"
+
+  role_name_prefix = "cluster-autoscaler"
+
+  attach_cluster_autoscaler_policy = true
+  cluster_autoscaler_cluster_names = [module.eks.cluster_name]
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:cluster-autoscaler"]
+    }
+  }
+
+  tags = local.tags
+}
diff --git a/terraform/environments/analytical-platform-compute/route53-zones.tf b/terraform/environments/analytical-platform-compute/route53-zones.tf
@@ -0,0 +1,16 @@
+# tflint-ignore: terraform_deprecated_interpolation
+module "route53_zones" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/route53/aws//modules/zones"
+  version = "2.11.1"
+
+  zones = {
+    "${local.environment_configuration.route53_zone}" = {
+      comment = local.environment_configuration.route53_zone
+    }
+  }
+
+  tags = local.tags
+}
diff --git a/...orm/environments/analytical-platform-compute/src/helm/cluster-autoscaler/values.yml.tftpl b/...orm/environments/analytical-platform-compute/src/helm/cluster-autoscaler/values.yml.tftpl
@@ -0,0 +1,20 @@
+---
+autoDiscovery:
+  cloudProviders: aws
+  clusterName: ${cluster_name}
+  tags:
+    - k8s.io/cluster-autoscaler/enabled
+    - k8s.io/cluster-autoscaler/${cluster_name}
+    - kubernetes.io/cluster/${cluster_name}
+
+awsRegion: ${aws_region}
+
+cloudProvider: aws
+
+fullnameOverride: cluster-autoscaler
+
+rbac:
+  serviceAccount:
+    name: cluster-autoscaler
+    annotations:
+      eks.amazonaws.com/role-arn: ${eks_role_arn}