Update Patch Manager to handle linux patching (#5479)

* Patch manager module changes

terraform/environments/hmpps-domain-services/locals_development.tf

@@ -88,6 +88,7 @@ locals {
         config = merge(module.baseline_presets.ec2_instance.config.default, {
           ami_name          = "base_rhel_8_5*"
           availability_zone = null
+          instance_profile_policies     = concat(module.baseline_presets.ec2_instance.config.default.instance_profile_policies, ["SSMPolicy", "PatchBucketAccessPolicy"])
         })
         instance = merge(module.baseline_presets.ec2_instance.instance.default, {
           vpc_security_group_ids = ["rds-ec2s"]

terraform/environments/hmpps-domain-services/locals_test.tf

@@ -87,6 +87,32 @@ locals {
           component   = "test"
         }
       }
+
+      test-rhel85 = {
+        config = merge(module.baseline_presets.ec2_instance.config.default, {
+          ami_name                  = "base_rhel_8_5*"
+          availability_zone         = null
+          instance_profile_policies = concat(module.baseline_presets.ec2_instance.config.default.instance_profile_policies, ["SSMPolicy", "PatchBucketAccessPolicy"])
+        })
+        instance = merge(module.baseline_presets.ec2_instance.instance.default, {
+          vpc_security_group_ids = ["rds-ec2s"]
+        })
+        user_data_cloud_init = merge(module.baseline_presets.ec2_instance.user_data_cloud_init.ssm_agent_and_ansible, {
+          args = merge(module.baseline_presets.ec2_instance.user_data_cloud_init.ssm_agent_and_ansible.args, {
+            branch = "main"
+          })
+        })
+        autoscaling_group = merge(module.baseline_presets.ec2_autoscaling_group.default, {
+          desired_capacity = 0
+        })
+        autoscaling_schedules = module.baseline_presets.ec2_autoscaling_schedules.working_hours
+        tags = {
+          description = "RHEL8.5"
+          ami         = "hmpps_rhel_8_5"
+          os-type     = "Linux"
+          component   = "test"
+        }
+      }
     }
 
     baseline_ec2_instances = {

terraform/environments/hmpps-domain-services/patch.tf

@@ -1,21 +1,25 @@
 module "development" {
-  count       = local.is-development == true ? 1 : 0
-  source      = "../../modules/patch_manager"
-  application = "hmpps-domain-services"
-  environment = "development"
-  schedule    = "cron(15 23 ? * * *)" # 11.15pm today
+  count               = local.is-development == true ? 1 : 0
+  source              = "../../modules/patch_manager"
+  application         = "hmpps-domain-services"
+  environment         = "development"
+  predefined_baseline = "AWS-RedHatDefaultPatchBaseline"
+  operating_system    = "REDHAT_ENTERPRISE_LINUX"
+  schedule            = "cron(15 23 ? * * *)" # 11.15pm today
   target_tag = {
     "environment-name" = "hmpps-domain-services-development"
   }
 }
 
 module "test" {
-  count       = local.is-test == true ? 1 : 0
-  source      = "../../modules/patch_manager"
-  application = "hmpps-domain-services"
-  environment = "test"
-  schedule    = "cron(15 23 ? * * *)" # 11.15pm today
+  count               = local.is-test == true ? 1 : 0
+  source              = "../../modules/patch_manager"
+  application         = "hmpps-domain-services"
+  environment         = "test"
+  predefined_baseline = "AWS-RedHatDefaultPatchBaseline"
+  operating_system    = "REDHAT_ENTERPRISE_LINUX"
+  schedule            = "cron(15 23 ? * * *)" # 11.15pm today
   target_tag = {
     "environment-name" = "hmpps-domain-services-test"
   }
-}
+}

terraform/modules/patch_manager/ExtractAndUploadPatches.yaml

@@ -3,7 +3,7 @@ description: "Extract successful patches and upload to s3 bucket"
 mainSteps:
   -
     action: "aws:runPowerShellScript"
-    name: "ExtractAndUploadPatches"
+    name: "ExtractAndUploadWindowsPatches"
     precondition:
       StringEquals:
         - platformType
@@ -57,4 +57,40 @@ mainSteps:
           }
           
           # Upload patch yaml to bucket
-          aws s3 cp $PatchesYaml  "$PatchesBucket/windows/${OS}Patches.yaml"
+          aws s3 cp $PatchesYaml  "$PatchesBucket/windows/${OS}Patches.yaml"
+  -
+    action: "aws:runShellScript"
+    name: "ExtractAndUploadLinuxPatches"
+    precondition:
+      StringEquals:
+        - platformType
+        - Linux
+    inputs:
+      runCommand:
+        - |
+          #!/bin/bash
+          set -euo pipefail
+          OS=$(hostnamectl | grep "Operating System:"|cut -d':' -f2- |tr -dc '[:alnum:]' )
+          PatchStatesYAMLPath="/tmp/$OS.yaml"
+          PatchStatesJSONPath="/var/log/amazon/ssm/patch-configuration/patch-states-configuration.json"
+          PatchBucket="s3://hmpps-domain-services-development-redhat-enterprise-linux"
+
+          # Install jq if not available
+          rpm -qa | grep -qw jq || sudo yum install -y  jq
+          
+          # Get all successfully installed patches
+          readarray -t patches < <(sudo  jq -r  '.patchStates | with_entries(select(.value.state != "Failed")) | keys[]' $PatchStatesJSONPath | sed 's/:/,*/')
+
+          # Construct a yaml file from the installed patches
+          echo "patches:" > $PatchStatesYAMLPath
+          for patch in "${patches[@]}"
+          do
+            id="${patch%%,*}"
+            title=${patch#*,}
+            echo "    -" >> $PatchStatesYAMLPath
+            echo "        id: '$id'" >> $PatchStatesYAMLPath
+            echo "        title: '$title'" >> $PatchStatesYAMLPath
+          done
+
+          # Upload patch yaml to s3
+          aws s3 cp $PatchStatesYAMLPath  "$PatchBucket/${OS}Patches.yaml"          

terraform/modules/patch_manager/README.md

@@ -3,13 +3,16 @@
 - Use this module to set up a patch schedule for instances.
 - Register instances to patch by giving the `target_tag` map the tag name and value of the instance that requires patching.
 - Successful patches will be picked up from the development environment and referenced by the other environments
+- Instances that require patching require the `PatchBucketAccessPolicy` added to their role. This role also needs to be trusted by the bucket in the module.
 
 ```hcl
 # Development environment will generate a list of patches
 module "development" {
   source                  = "./modules/"
   application             = "hmpps-domain-services"
   environment             = "development"
+  predefined_baseline     = "AWS-WindowsPredefinedPatchBaseline-OS-Applications"
+  operating_system        = "WINDOWS"
   # Second Tuesday of the month at 9pm UTC
   schedule                = "cron(0 21 ? * TUE#2 *)"
   target_tag              =  {
@@ -22,10 +25,11 @@ module "test" {
   source                  = "./modules/"
   application             = "hmpps-domain-services"
   environment             = "test"
+  predefined_baseline     = "AWS-WindowsPredefinedPatchBaseline-OS-Applications"
+  operating_system        = "WINDOWS"  
   schedule                = "cron(0 21 ? * WED#2 *)"
   target_tag              =  {
     "environment-name" = "hmpps-domain-services-test"
   }
  }
-```
-
+```

terraform/modules/patch_manager/patch-manager.tf

@@ -10,17 +10,17 @@ resource "aws_ssm_default_patch_baseline" "this" {
 }
 
 resource "aws_ssm_maintenance_window" "this" {
-  name        = "${var.application}-${var.environment}-maintenance-window"
+  name        = "${var.application}-${var.environment}-${local.os}-maintenance-window"
   schedule    = var.schedule
-  description = "Maintenance window for ${var.application}-${var.environment}"
+  description = "${var.application}-${var.environment}-${local.os} maintenance window"
   duration    = 3
   cutoff      = 1
 }
 
 resource "aws_ssm_maintenance_window_target" "this" {
   window_id     = aws_ssm_maintenance_window.this.id
   resource_type = "INSTANCE"
-  description   = "${var.application}-${var.environment} target"
+  description   = "${var.application}-${var.environment}-${local.os} target"
 
   targets {
     key    = "tag:${keys(var.target_tag)[0]}"
@@ -29,12 +29,12 @@ resource "aws_ssm_maintenance_window_target" "this" {
 }
 
 resource "aws_cloudwatch_log_group" "this" {
-  name              = "${var.application}-${var.environment}-patch-logs"
+  name              = "${var.application}-${var.environment}-${local.os}-patch-logs"
   retention_in_days = 30
 }
 
 resource "aws_ssm_maintenance_window_task" "this" {
-  description = "Maintenance window task for ${var.application}-${var.environment}"
+  description = "Patching task for ${var.application}-${var.environment}-${local.os}"
   task_type   = "RUN_COMMAND"
   # Only development uses AWS-RunPatchBaselineWithHooks to trigger post patching jobs and you can't use this task
   # when specifying exact patches so the environments will run the standard AWS-RunPatchBaseline task
@@ -72,21 +72,20 @@ resource "aws_ssm_maintenance_window_task" "this" {
         values = ["RebootIfNeeded"]
       }
 
+      # All non-development environments pull patch list from development
       dynamic "parameter" {
-        for_each = var.environment == "development" ? [1] : []
+        for_each = var.environment != "development" && var.operating_system == "WINDOWS" ? [1] : []
         content {
-          # Extract successful patches after development gets patched
-          name   = "PostInstallHookDocName"
-          values = [aws_ssm_document.extract-upload-patches[0].arn]
+          name   = "InstallOverrideList"
+          values = ["s3://${var.application}-development-${local.os}/WindowsServer2022DatacenterPatches.yaml"]
         }
       }
 
       dynamic "parameter" {
-        for_each = var.environment != "development" ? [1] : []
+        for_each = var.environment != "development" && var.operating_system == "REDHAT_ENTERPRISE_LINUX" ? [1] : []
         content {
-          # All non-development environments pull patch list from development
           name   = "InstallOverrideList"
-          values = ["s3://${var.application}-development-patch-logs/windows/WindowsServer2022DatacenterPatches.yaml"]
+          values = ["s3://${var.application}-development-${local.os}/RedHatEnterpriseLinux89OotpaPatches.yaml"]
         }
       }
     }

terraform/modules/patch_manager/s3.tf

@@ -1,5 +1,5 @@
 resource "aws_s3_bucket" "this" {
-  bucket        = "${var.application}-${var.environment}-patch-logs"
+  bucket        = "${var.application}-${var.environment}-${local.os}"
   force_destroy = true
 }
 
@@ -27,6 +27,7 @@ data "aws_iam_policy_document" "bucket_policy_patch_access" {
     principals {
       identifiers = [
         "arn:aws:iam::139351334100:role/ec2-instance-role-dev-win-2022",
+        "arn:aws:iam::139351334100:role/ec2-instance-role-dev-rhel85",
         "arn:aws:iam::161282055413:role/ec2-instance-role-test-win-2022",
         "arn:aws:iam::228371063224:role/ec2-instance-role-pp-rdgw-1-a",
         "arn:aws:iam::228371063224:role/ec2-instance-role-pp-rds-1-a",

terraform/modules/patch_manager/variables.tf

@@ -11,15 +11,35 @@ variable "schedule" {
 }
 
 variable "predefined_baseline" {
-  type    = string
-  default = "AWS-WindowsPredefinedPatchBaseline-OS-Applications"
+  description = "Predefined baseline"
+  type        = string
+  validation {
+    condition = contains([
+      "AWS-WindowsPredefinedPatchBaseline-OS",
+      "AWS-WindowsPredefinedPatchBaseline-OS-Applications",
+      "AWS-RedHatDefaultPatchBaseline"
+    ], var.predefined_baseline)
+    error_message = "Not a valid baseline"
+  }
 }
 
 variable "operating_system" {
-  type    = string
-  default = "WINDOWS"
+  description = "Operating system for baseline"
+  type        = string
+  validation {
+    condition = contains([
+      "WINDOWS",
+      "REDHAT_ENTERPRISE_LINUX"
+    ], var.operating_system)
+    error_message = "Not a valid operating system"
+  }
 }
 
 variable "target_tag" {
-  type = map(any)
+  description = "Instance tag name and value to target for patching"
+  type        = map(any)
+}
+
+locals {
+  os = replace(lower(var.operating_system), "_", "-")
 }