TM-720: add ssm endpoint monitoring to hmpp-oem (#8763)

* add enable_ssm_command_monitoring option

* enable ssm command monitoring

terraform/environments/hmpps-oem/locals.tf

@@ -49,6 +49,7 @@ locals {
       enable_s3_db_backup_bucket                  = true
       enable_s3_shared_bucket                     = true
       enable_s3_software_bucket                   = true
+      enable_ssm_command_monitoring               = true
       s3_iam_policies                             = ["EC2S3BucketWriteAndDeleteAccessPolicy"]
     }
   }

terraform/environments/hmpps-oem/main.tf

@@ -74,6 +74,7 @@ module "baseline" {
   )
 
   cloudwatch_metric_alarms = merge(
+    module.baseline_presets.cloudwatch_metric_alarms_baseline,
     lookup(local.baseline_all_environments, "cloudwatch_metric_alarms", {}),
     lookup(local.baseline_environment_specific, "cloudwatch_metric_alarms", {}),
   )

terraform/modules/baseline_presets/cloudwatch_metric_alarms.tf

@@ -449,4 +449,25 @@ locals {
       }
     }
   }
+
+  cloudwatch_metric_alarms_by_sns_topic = {
+    for sns_key, sns_value in local.sns_topics : sns_key => {
+      for namespace_key, namespace_value in local.cloudwatch_metric_alarms : namespace_key => {
+        for alarm_key, alarm_value in namespace_value : alarm_key => merge(alarm_value, {
+          alarm_actions = [sns_key]
+          ok_actions    = [sns_key]
+        })
+      }
+    }
+  }
+
+  # alarms added via baseline. Put SSM command alerts in dso-pipelines so it doesn't clutter main application alerts
+  cloudwatch_metric_alarms_baseline = merge(
+    var.options.enable_ssm_command_monitoring ? {
+      "failed-ssm-command-${var.environment.account_name}" = local.cloudwatch_metric_alarms_by_sns_topic["dso-pipelines-pagerduty"].ssm.failed-ssm-command
+    } : {},
+    var.options.enable_ssm_command_monitoring ? {
+      "ssm-command-metrics-missing-${var.environment.account_name}" = local.cloudwatch_metric_alarms_by_sns_topic["dso-pipelines-pagerduty"].ssm.ssm-command-metrics-missing
+    } : {},
+  )
 }

terraform/modules/baseline_presets/outputs.tf

@@ -44,19 +44,15 @@ output "cloudwatch_metric_alarms" {
   value       = local.cloudwatch_metric_alarms
 }
 
+output "cloudwatch_metric_alarms_baseline" {
+  description = "Map of common cloudwatch metric alarms that can be passed into baseline directly as specified by var.options.enable_ssm_command_monitoring for example"
+  value       = local.cloudwatch_metric_alarms_baseline
+}
+
 output "cloudwatch_metric_alarms_by_sns_topic" {
   description = "Map of sns topic key to cloudwatch metric alarms grouped by namespace, where the default action is the sns topic key"
 
-  value = {
-    for sns_key, sns_value in local.sns_topics : sns_key => {
-      for namespace_key, namespace_value in local.cloudwatch_metric_alarms : namespace_key => {
-        for alarm_key, alarm_value in namespace_value : alarm_key => merge(alarm_value, {
-          alarm_actions = [sns_key]
-          ok_actions    = [sns_key]
-        })
-      }
-    }
-  }
+  value = local.cloudwatch_metric_alarms_by_sns_topic
 }
 
 output "iam_roles" {

terraform/modules/baseline_presets/sns_topics.tf

@@ -6,8 +6,14 @@
 #   from the modernisation platform managed pagerduty_integration_keys
 
 locals {
+
+  pagerduty_integrations = merge(
+    var.options.enable_ssm_command_monitoring ? { dso-pipelines-pagerduty = "dso-pipelines" } : {},
+    var.options.sns_topics.pagerduty_integrations
+  )
+
   sns_topics_pagerduty_integrations = {
-    for key, value in var.options.sns_topics.pagerduty_integrations : key => {
+    for key, value in local.pagerduty_integrations : key => {
       display_name      = "Pager duty integration for ${value}"
       kms_master_key_id = "general"
       subscriptions = {

terraform/modules/baseline_presets/variables.tf

@@ -42,6 +42,7 @@ variable "options" {
     enable_s3_db_backup_bucket                   = optional(bool, false)           # create db-backup S3 buckets
     enable_s3_shared_bucket                      = optional(bool, false)           # create devtest and preprodprod S3 bucket for sharing between accounts
     enable_s3_software_bucket                    = optional(bool, false)           # create software S3 bucket in test account for image builder/configuration-management
+    enable_ssm_command_monitoring                = optional(bool, false)           # create SNS topic and alarms for SSM command monitoring
     enable_vmimport                              = optional(bool, false)           # create role for vm imports
     route53_resolver_rules                       = optional(map(list(string)), {}) # create route53 resolver rules; list of map keys to filter local.route53_resolver_rules_all
     iam_service_linked_roles                     = optional(list(string))          # create iam service linked roles; list of map keys to filter local.iam_service_linked_roles; default is to create all