Merge branch 'main' into feat/ap-ingest-datasync-task

ministryofjustice · Dec 6, 2024 · d335453 · d335453
2 parents 305fb4e + ff87a99
commit d335453
Show file tree

Hide file tree

Showing 255 changed files with 8,433 additions and 3,315 deletions.
diff --git a/.devcontainer/README.md b/.devcontainer/README.md
@@ -1,36 +1,37 @@
 # Dev Container
 
+> [!NOTE]
 > This is a community supported feature
 
-To assist in the development of `modernisation-platform-environments`, the community have built a [dev container](https://containers.dev/) with the required tooling
+To assist with working on this repository, the community has configured a [dev container](https://containers.dev/) with the required tooling.
 
-## Prerequisites
+You can run this locally, or with [GitHub Codespaces](https://docs.github.com/en/codespaces/overview).
 
-- GitHub Codespaces
+## Locally
 
-or
+> [!WARNING]  
+> This has only been tested on macOS
+
+### Prerequisites
 
 - Docker
 
 - Visual Studio Code
 
   - Dev Containers Extention
 
-## Running
-
-### GitHub Codespaces
-
-Launch from GitHub
+To launch locally, ensure the prerequisites are met, and then click the button below
 
-### Locally
+[![Open in Dev Container](https://raw.githubusercontent.com/ministryofjustice/.devcontainer/refs/heads/main/contrib/badge.svg)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/ministryofjustice/modernisation-platform-environments)
 
-1. Ensure prerequisites are met
+## GitHub Codespaces
 
-1. Clone repository
+> [!IMPORTANT]  
+> GitHub Codespaces are not currently paid for by the Ministry of Justice and are subject to the quotas [here](https://docs.github.com/en/billing/managing-billing-for-your-products/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts)
 
-1. Open repository in Visual Studio Code
+To launch a GitHub Codespace, click the button below
 
-1. Reopen in container
+[![Open in Codespace](https://github.com/codespaces/badge.svg)](https://codespaces.new/ministryofjustice/modernisation-platform-environments)
 
 ## Tools
 

diff --git a/.github/workflows/awsnuke.yml b/.github/workflows/awsnuke.yml
@@ -133,11 +133,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       ACCOUNT_NAME: ${{ matrix.nuke_accts }}
@@ -217,11 +217,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       AWS_ACCESS_KEY_ID:  ${{ secrets.TESTING_AWS_ACCESS_KEY_ID }}

diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml
@@ -38,7 +38,7 @@ jobs:
         run: tflint --disable-rule=terraform_unused_declarations --format sarif > tflint.sarif
       - name: Upload SARIF file
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: tflint.sarif
   trivy:
@@ -53,7 +53,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run Trivy vulnerability scanner in repo mode
-        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           scan-type: 'fs'
           scanners: misconfig,vuln,secret
@@ -63,7 +63,7 @@ jobs:
 
       - name: Upload Trivy scan results to GitHub Security tab
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: 'trivy-results.sarif'
   checkov:
@@ -81,7 +81,7 @@ jobs:
           fetch-depth: 0
       - name: Run Checkov action
         id: checkov
-        uses: bridgecrewio/checkov-action@6fe02213c515948c8da243a6554a3bff49129295 # v12.2912.0
+        uses: bridgecrewio/checkov-action@f10397402800d31940c9cefd680c66688a516c9f # v12.2932.0
         with:
           directory: ./
           framework: terraform
@@ -90,6 +90,6 @@ jobs:
           skip_check: CKV_GIT_1,CKV_AWS_126,CKV2_AWS_38,CKV2_AWS_39
       - name: Upload SARIF file
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: ./checkov.sarif
diff --git a/.github/workflows/format-code.yml b/.github/workflows/format-code.yml
@@ -40,7 +40,7 @@ jobs:
         id: ml
         # You can override MegaLinter flavor used to have faster performances
         # More info at https://megalinter.io/flavors/
-        uses: oxsecurity/megalinter/flavors/terraform@d8c95fc6f2237031fb9e9322b0f97100168afa6e #v8.2.0
+        uses: oxsecurity/megalinter/flavors/terraform@1fc052d03c7a43c78fe0fee19c9d648b749e0c01 #v8.3.0
         env:
           # All available variables are described in documentation
           # https://megalinter.io/configuration/#shared-variables

diff --git a/.github/workflows/nuke-redeploy.yml b/.github/workflows/nuke-redeploy.yml
@@ -93,11 +93,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
 
     env:

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: results.sarif
diff --git a/terraform/environments/analytical-platform-compute/ec2-instances.tf b/terraform/environments/analytical-platform-compute/ec2-instances.tf
@@ -0,0 +1,35 @@
+module "debug_instance" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/ec2-instance/aws"
+  version = "5.7.1"
+
+  name                        = "network-debug"
+  ami                         = "ami-0e8d228ad90af673b" # Ubuntu Server 24.04 LTS
+  instance_type               = "t3.micro"
+  subnet_id                   = element(module.vpc.private_subnets, 0)
+  vpc_security_group_ids      = [module.debug_instance_security_group.security_group_id]
+  associate_public_ip_address = false
+
+  root_block_device = [
+    {
+      encrypted   = true
+      volume_type = "gp3"
+      volume_size = 8
+    }
+  ]
+
+  create_iam_instance_profile = true
+  iam_role_policies = {
+    SSMCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  }
+
+  metadata_options = {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "required"
+    instance_metadata_tags      = "enabled"
+  }
+
+  tags = local.tags
+}
diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf
@@ -13,11 +13,7 @@ locals {
       vpc_single_nat_gateway     = false
 
       /* Transit Gateway */
-      transit_gateway_routes = [
-        "10.26.0.0/15", # modernisation-platform
-        "10.40.0.0/18", # noms-live-vnet
-        "10.205.0.0/20" # laa-lz-prod
-      ]
+      transit_gateway_routes = ["10.0.0.0/8"]
 
       /* Route53 */
       route53_zone = "compute.development.analytical-platform.service.justice.gov.uk"
@@ -42,14 +38,12 @@ locals {
       /* MLFlow */
       mlflow_s3_bucket_name = "alpha-analytical-platform-mlflow-development"
 
-      /* Observability Platform */
-      observability_platform = "development"
-
       /* QuickSight */
       quicksight_notification_email = "analytical-platform@digital.justice.gov.uk"
 
       /* UI */
       ui_hostname = "development.analytical-platform.service.justice.gov.uk"
+
     }
     test = {
       /* VPC */
@@ -64,11 +58,7 @@ locals {
       vpc_single_nat_gateway     = false
 
       /* Transit Gateway */
-      transit_gateway_routes = [
-        "10.26.0.0/15", # modernisation-platform
-        "10.40.0.0/18", # noms-live-vnet
-        "10.205.0.0/20" # laa-lz-prod
-      ]
+      transit_gateway_routes = ["10.0.0.0/8"]
 
       /* Route53 */
       route53_zone = "compute.test.analytical-platform.service.justice.gov.uk"
@@ -87,9 +77,6 @@ locals {
         vpc_cni                = "v1.19.0-eksbuild.1"
       }
 
-      /* Observability Platform */
-      observability_platform = "development"
-
       /* Data Engineering Airflow */
       data_engineering_airflow_execution_role_arn = "arn:aws:iam::${local.environment_management.account_ids["analytical-platform-data-production"]}:role/airflow-dev-execution-role"
 
@@ -115,10 +102,7 @@ locals {
       vpc_single_nat_gateway     = false
 
       /* Transit Gateway */
-      transit_gateway_routes = [
-        "10.26.0.0/15", # modernisation-platform
-        "10.40.0.0/18"  # noms-live-vnet
-      ]
+      transit_gateway_routes = ["10.0.0.0/8"]
 
       /* Route53 */
       route53_zone = "compute.analytical-platform.service.justice.gov.uk"
@@ -143,14 +127,16 @@ locals {
       /* MLFlow */
       mlflow_s3_bucket_name = "alpha-analytical-platform-mlflow"
 
-      /* Observability Platform */
-      observability_platform = "production"
-
       /* QuickSight */
       quicksight_notification_email = "analytical-platform@digital.justice.gov.uk"
 
       /* UI */
       ui_hostname = "analytical-platform.service.justice.gov.uk"
+
+      /* LF Domain Tags */
+      cadet_lf_tags = {
+        domain = ["bold", "civil", "courts", "general", "criminal_history", "development_sandpit", "electronic_monitoring", "finance", "interventions", "opg", "performance", "risk", "people", "prison", "probation", "staging", "victims", "victims_case_management"] # extracted from bucket paths
+      }
     }
   }
 }
diff --git a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf
@@ -174,3 +174,27 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_emds_test"
     )
   ]
 }
+
+resource "helm_release" "actions_runner_mojas_create_a_derived_table_emds" {
+  count = terraform.workspace == "analytical-platform-compute-production" ? 1 : 0
+
+  /* https://github.com/ministryofjustice/analytical-platform-actions-runner */
+  name       = "actions-runner-mojas-create-a-derived-table-emds"
+  repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts"
+  version    = "2.320.0-4"
+  chart      = "actions-runner"
+  namespace  = kubernetes_namespace.actions_runners[0].metadata[0].name
+  values = [
+    templatefile(
+      "${path.module}/src/helm/values/actions-runners/create-a-derived-table/values.yml.tftpl",
+      {
+        github_app_application_id  = jsondecode(data.aws_secretsmanager_secret_version.actions_runners_token_apc_self_hosted_runners_github_app[0].secret_string)["app_id"]
+        github_app_installation_id = jsondecode(data.aws_secretsmanager_secret_version.actions_runners_token_apc_self_hosted_runners_github_app[0].secret_string)["installation_id"]
+        github_organisation        = "moj-analytical-services"
+        github_repository          = "create-a-derived-table"
+        github_runner_labels       = "electronic-monitoring-data"
+        eks_role_arn               = "arn:aws:iam::${local.environment_management.account_ids["electronic-monitoring-data-production"]}:role/prod-data-api-cross-account-role"
+      }
+    )
+  ]
+}
diff --git a/terraform/environments/analytical-platform-compute/helm-charts-applications.tf b/terraform/environments/analytical-platform-compute/helm-charts-applications.tf