From 105171c1b01b1f5b600ca292775d911d21812c23 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 12:33:58 +0100 Subject: [PATCH 01/24] add cloudwatch metric alarm notification on load balancer --- terraform/environments/cdpt-ifs/monitoring.tf | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 terraform/environments/cdpt-ifs/monitoring.tf diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf new file mode 100644 index 00000000000..db935b92415 --- /dev/null +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -0,0 +1,25 @@ +resource "aws_sns_topic" "lb_alarm_topic" { + name = "lb_alarm_topic" +} + +resource "aws_sns_topic_subscription" "slack_subscription" { + topic_arn = aws_sns_topic.lb_alarm_topic.arn + protocol = "https" + endpoint = "https://hooks.slack.com/services/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" +} + +resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { + alarm_name = "lb-5xx-errors" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = "1" + metric_name = "HTTPCode_ELB_5XX_Count" + namespace = "AWS/ApplicationELB" + period = "300" + statistic = "Sum" + threshold = "1" + alarm_description = "This metric monitors 5xx errors on the load balancer" + alarm_actions = [aws_sns_topic.lb_alarm_topic.arn] + + dimensions = { + LoadBalancer = "${var.application_name}-lb" +} From 9fe1b552d4b307134695a6e3e21941bb6d00bdfe Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 12:35:58 +0100 Subject: [PATCH 02/24] fix missing } --- terraform/environments/cdpt-ifs/monitoring.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index db935b92415..583c73f8990 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -22,4 +22,5 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { dimensions = { LoadBalancer = "${var.application_name}-lb" + } } From 285d20f5fb6e0574de694aaa1a04f496f8deb5de Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 12:42:48 +0100 Subject: [PATCH 03/24] fix incorrect application name --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 583c73f8990..d69aa3d613f 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -21,6 +21,6 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_actions = [aws_sns_topic.lb_alarm_topic.arn] dimensions = { - LoadBalancer = "${var.application_name}-lb" + LoadBalancer = "${local.application_name}-lb" } } From 8ee77a57e95e78e6ad09ad16b9590ff49395ab96 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 14:04:44 +0100 Subject: [PATCH 04/24] trigger lb alarm with incorrect healthcheck path --- terraform/environments/cdpt-ifs/loadbalancer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/loadbalancer.tf b/terraform/environments/cdpt-ifs/loadbalancer.tf index 8fe06aa62fc..8d34be56467 100644 --- a/terraform/environments/cdpt-ifs/loadbalancer.tf +++ b/terraform/environments/cdpt-ifs/loadbalancer.tf @@ -118,7 +118,7 @@ resource "aws_lb_target_group" "ifs_target_group" { unhealthy_threshold = "2" matcher = "200-499" timeout = "15" - path = "/health" + path = "/test-lb-alert" } lifecycle { From d30a11bdd696d88c91ae27a45bbf1508410dc099 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 14:16:58 +0100 Subject: [PATCH 05/24] trigger lb alarm with 0 targets --- terraform/environments/cdpt-ifs/application_variables.json | 4 ++-- terraform/environments/cdpt-ifs/loadbalancer.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/cdpt-ifs/application_variables.json b/terraform/environments/cdpt-ifs/application_variables.json index ce638c15393..1a785338770 100644 --- a/terraform/environments/cdpt-ifs/application_variables.json +++ b/terraform/environments/cdpt-ifs/application_variables.json @@ -6,9 +6,9 @@ "client_id": "7ee6af8d-ea3c-4349-8765-644f2a1edf3b", "instance_type": "t3.xlarge", "app_count": 1, - "ec2_desired_capacity": 1, + "ec2_desired_capacity": 0, "ec2_max_size": 3, - "ec2_min_size": 1, + "ec2_min_size": 0, "db_instance_class": "db.t3.small", "db_user": "dbadmin", "db_allocated_storage": "100", diff --git a/terraform/environments/cdpt-ifs/loadbalancer.tf b/terraform/environments/cdpt-ifs/loadbalancer.tf index 8d34be56467..8fe06aa62fc 100644 --- a/terraform/environments/cdpt-ifs/loadbalancer.tf +++ b/terraform/environments/cdpt-ifs/loadbalancer.tf @@ -118,7 +118,7 @@ resource "aws_lb_target_group" "ifs_target_group" { unhealthy_threshold = "2" matcher = "200-499" timeout = "15" - path = "/test-lb-alert" + path = "/health" } lifecycle { From 05ff89a100f8bfa9b6fb00e843c658e41424e140 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Thu, 4 Jul 2024 15:45:35 +0100 Subject: [PATCH 06/24] reset desired targets --- terraform/environments/cdpt-ifs/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/cdpt-ifs/application_variables.json b/terraform/environments/cdpt-ifs/application_variables.json index 1a785338770..ce638c15393 100644 --- a/terraform/environments/cdpt-ifs/application_variables.json +++ b/terraform/environments/cdpt-ifs/application_variables.json @@ -6,9 +6,9 @@ "client_id": "7ee6af8d-ea3c-4349-8765-644f2a1edf3b", "instance_type": "t3.xlarge", "app_count": 1, - "ec2_desired_capacity": 0, + "ec2_desired_capacity": 1, "ec2_max_size": 3, - "ec2_min_size": 0, + "ec2_min_size": 1, "db_instance_class": "db.t3.small", "db_user": "dbadmin", "db_allocated_storage": "100", From 9c9c42ad28b1c940ea38debe6cc35d8aa1051137 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 12:26:25 +0100 Subject: [PATCH 07/24] add cloudwatch alert using module --- .../cdpt-ifs/application_variables.json | 9 +++-- terraform/environments/cdpt-ifs/monitoring.tf | 38 ++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/terraform/environments/cdpt-ifs/application_variables.json b/terraform/environments/cdpt-ifs/application_variables.json index ce638c15393..3e9e5a1bcea 100644 --- a/terraform/environments/cdpt-ifs/application_variables.json +++ b/terraform/environments/cdpt-ifs/application_variables.json @@ -16,7 +16,8 @@ "env_name": "DEVELOPMENT", "db_instance_identifier": "db-ifs-dev", "s3_bucket_name": "ifs-dev-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:425030398367:snapshot:dev-staging-db-11-june-24-updated" + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:425030398367:snapshot:dev-staging-db-11-june-24-updated", + "enable_slack_alerts": true, }, "preproduction": { "environment_name": "staging", @@ -34,7 +35,8 @@ "env_name": "STAGING", "db_instance_identifier": "db-ifs-staging", "s3_bucket_name": "ifs-staging-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-cdpt-dev-staging-feb-12-2024" + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-cdpt-dev-staging-feb-12-2024", + "enable_slack_alerts": true, }, "production": { "environment_name": "production", @@ -52,7 +54,8 @@ "env_name": "PRODUCTION", "db_instance_identifier": "db-ifs-production", "s3_bucket_name": "ifs-production-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-production-db-02-04-2024" + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-production-db-02-04-2024", + "enable_slack_alerts": true, } } } diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index d69aa3d613f..257f93c5eeb 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -2,10 +2,40 @@ resource "aws_sns_topic" "lb_alarm_topic" { name = "lb_alarm_topic" } +module "slack_alerts_url" { + for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} + source = "./modules/secrets_manager" + name = "${local.application_name}-slack-alerts-url-${local.environment}" + description = "IFS LoadBalancer slack alerts URL" + type = "MONO" + secret_value = "http://Placeholder_webhook_URL" + ignore_secret_string = true + + tags = merge( + local.all_tags, + { + Resource_Type = "Secret" + Name = "${local.application_name}-slack-alerts-url-${local.environment}" + } + ) +} + +data "aws_secretsmanager_secret" "slack_integration" { + for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} + depends_on = [module.slack_alerts_url] + name = "${local.project}-slack-alerts-url-${local.environment}" +} + +data "aws_secretsmanager_secret_version" "slack_integration" { + for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} + secret_id = data.aws_secretsmanager_secret.slack_integration[0].id +} + resource "aws_sns_topic_subscription" "slack_subscription" { + depends_on = [data.aws_secretsmanager_secret_version.slack_integration] topic_arn = aws_sns_topic.lb_alarm_topic.arn protocol = "https" - endpoint = "https://hooks.slack.com/services/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" + endpoint = data.aws_secretsmanager_secret_version.slack_integration["enabled"].secret_string } resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { @@ -24,3 +54,9 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { LoadBalancer = "${local.application_name}-lb" } } + + + + + + From e03f297c54e57b1bcd61aa93477ea67f51ac5ab8 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 13:24:51 +0100 Subject: [PATCH 08/24] fix module path --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 257f93c5eeb..5b53aa0f400 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -4,7 +4,7 @@ resource "aws_sns_topic" "lb_alarm_topic" { module "slack_alerts_url" { for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} - source = "./modules/secrets_manager" + source = "./modules/baseline/secrets_manager" name = "${local.application_name}-slack-alerts-url-${local.environment}" description = "IFS LoadBalancer slack alerts URL" type = "MONO" From d0bc6438d0892fded048ea63851711c5b3856ff8 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 13:29:08 +0100 Subject: [PATCH 09/24] fix module path --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 5b53aa0f400..b0c240de04a 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -4,7 +4,7 @@ resource "aws_sns_topic" "lb_alarm_topic" { module "slack_alerts_url" { for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} - source = "./modules/baseline/secrets_manager" + source = "./modules/baseline/secretsmanager" name = "${local.application_name}-slack-alerts-url-${local.environment}" description = "IFS LoadBalancer slack alerts URL" type = "MONO" From 94fd050e09b466cd78f6e38882c593a35df686c9 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 14:56:58 +0100 Subject: [PATCH 10/24] add cw alarm with pagerduty integration --- .../cdpt-ifs/application_variables.json | 9 +-- terraform/environments/cdpt-ifs/monitoring.tf | 68 ++++++++----------- 2 files changed, 30 insertions(+), 47 deletions(-) diff --git a/terraform/environments/cdpt-ifs/application_variables.json b/terraform/environments/cdpt-ifs/application_variables.json index 3e9e5a1bcea..ce638c15393 100644 --- a/terraform/environments/cdpt-ifs/application_variables.json +++ b/terraform/environments/cdpt-ifs/application_variables.json @@ -16,8 +16,7 @@ "env_name": "DEVELOPMENT", "db_instance_identifier": "db-ifs-dev", "s3_bucket_name": "ifs-dev-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:425030398367:snapshot:dev-staging-db-11-june-24-updated", - "enable_slack_alerts": true, + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:425030398367:snapshot:dev-staging-db-11-june-24-updated" }, "preproduction": { "environment_name": "staging", @@ -35,8 +34,7 @@ "env_name": "STAGING", "db_instance_identifier": "db-ifs-staging", "s3_bucket_name": "ifs-staging-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-cdpt-dev-staging-feb-12-2024", - "enable_slack_alerts": true, + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-cdpt-dev-staging-feb-12-2024" }, "production": { "environment_name": "production", @@ -54,8 +52,7 @@ "env_name": "PRODUCTION", "db_instance_identifier": "db-ifs-production", "s3_bucket_name": "ifs-production-bucket", - "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-production-db-02-04-2024", - "enable_slack_alerts": true, + "db_snapshot_identifier": "arn:aws:rds:eu-west-2:613903586696:snapshot:ifs-production-db-02-04-2024" } } } diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index b0c240de04a..abf80dfb61e 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -1,43 +1,3 @@ -resource "aws_sns_topic" "lb_alarm_topic" { - name = "lb_alarm_topic" -} - -module "slack_alerts_url" { - for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} - source = "./modules/baseline/secretsmanager" - name = "${local.application_name}-slack-alerts-url-${local.environment}" - description = "IFS LoadBalancer slack alerts URL" - type = "MONO" - secret_value = "http://Placeholder_webhook_URL" - ignore_secret_string = true - - tags = merge( - local.all_tags, - { - Resource_Type = "Secret" - Name = "${local.application_name}-slack-alerts-url-${local.environment}" - } - ) -} - -data "aws_secretsmanager_secret" "slack_integration" { - for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} - depends_on = [module.slack_alerts_url] - name = "${local.project}-slack-alerts-url-${local.environment}" -} - -data "aws_secretsmanager_secret_version" "slack_integration" { - for_each = local.enable_slack_alerts ? { "enabled": "enabled" } : {} - secret_id = data.aws_secretsmanager_secret.slack_integration[0].id -} - -resource "aws_sns_topic_subscription" "slack_subscription" { - depends_on = [data.aws_secretsmanager_secret_version.slack_integration] - topic_arn = aws_sns_topic.lb_alarm_topic.arn - protocol = "https" - endpoint = data.aws_secretsmanager_secret_version.slack_integration["enabled"].secret_string -} - resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_name = "lb-5xx-errors" comparison_operator = "GreaterThanOrEqualToThreshold" @@ -49,14 +9,40 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { threshold = "1" alarm_description = "This metric monitors 5xx errors on the load balancer" alarm_actions = [aws_sns_topic.lb_alarm_topic.arn] - dimensions = { LoadBalancer = "${local.application_name}-lb" } } +resource "aws_sns_topic" "lb_5xx_alarm_topic" { + name = "lb_5xx_alarm_topic" + kms_master_key_id = data.aws_kms_key.sns.id +} +# Pager duty integration +# Get the map of pagerduty integration keys from the modernisation platform account +data "aws_secretsmanager_secret" "pagerduty_integration_keys" { + provider = aws.modernisation-platform + name = "pagerduty_integration_keys" +} +data "aws_secretsmanager_secret_version" "pagerduty_integration_keys" { + provider = aws.modernisation-platform + secret_id = data.aws_secretsmanager_secret.pagerduty_integration_keys.id +} +# Add a local to get the keys +locals { + pagerduty_integration_keys = jsondecode(data.aws_secretsmanager_secret_version.pagerduty_integration_keys.secret_string) +} +# link the sns topic to the service +module "pagerduty_core_alerts" { + depends_on = [ + aws_sns_topic.lb-5xx-errors + ] + source = "github.com/ministryofjustice/modernisation-platform-terraform-pagerduty-integration?ref=v2.0.0" + sns_topics = [aws_sns_topic.lb_5xx_alarm_topic.name] + pagerduty_integration_key = local.pagerduty_integration_keys["cloudwatch_lb_alert"] +} From 914fb5d5e7a0f01713416956155c3ac551b3cf9d Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 16:17:48 +0100 Subject: [PATCH 11/24] fix typos --- terraform/environments/cdpt-ifs/monitoring.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index abf80dfb61e..371d4a0cddd 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -8,7 +8,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { statistic = "Sum" threshold = "1" alarm_description = "This metric monitors 5xx errors on the load balancer" - alarm_actions = [aws_sns_topic.lb_alarm_topic.arn] + alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { LoadBalancer = "${local.application_name}-lb" } @@ -16,7 +16,6 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { resource "aws_sns_topic" "lb_5xx_alarm_topic" { name = "lb_5xx_alarm_topic" - kms_master_key_id = data.aws_kms_key.sns.id } # Pager duty integration From 3b5594d94af92f49d20e3be5a4dd8d1737221812 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 16:20:12 +0100 Subject: [PATCH 12/24] fix typos --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 371d4a0cddd..b80ff34a463 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -38,7 +38,7 @@ locals { # link the sns topic to the service module "pagerduty_core_alerts" { depends_on = [ - aws_sns_topic.lb-5xx-errors + aws_sns_topic.lb_5xx_alarm_topic ] source = "github.com/ministryofjustice/modernisation-platform-terraform-pagerduty-integration?ref=v2.0.0" sns_topics = [aws_sns_topic.lb_5xx_alarm_topic.name] From 6f362ac36ff259a5e242dc06afc182ad5b6f84fd Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Fri, 5 Jul 2024 16:35:03 +0100 Subject: [PATCH 13/24] add correct integration key --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index b80ff34a463..217d5149dae 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -42,6 +42,6 @@ module "pagerduty_core_alerts" { ] source = "github.com/ministryofjustice/modernisation-platform-terraform-pagerduty-integration?ref=v2.0.0" sns_topics = [aws_sns_topic.lb_5xx_alarm_topic.name] - pagerduty_integration_key = local.pagerduty_integration_keys["cloudwatch_lb_alert"] + pagerduty_integration_key = local.pagerduty_integration_keys["cdpt-ifs-alarms"] } From e58be5f7b60f7fef203f57ad2434f339e039c407 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Mon, 8 Jul 2024 15:10:02 +0100 Subject: [PATCH 14/24] use target 5xx error instead --- terraform/environments/cdpt-ifs/monitoring.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 217d5149dae..4c005aa8302 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -2,12 +2,12 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_name = "lb-5xx-errors" comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = "1" - metric_name = "HTTPCode_ELB_5XX_Count" + metric_name = "HTTPCode_Target_5XX_Count" namespace = "AWS/ApplicationELB" period = "300" statistic = "Sum" threshold = "1" - alarm_description = "This metric monitors 5xx errors on the load balancer" + alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { LoadBalancer = "${local.application_name}-lb" From aa07430f768d461257cb26f03f3706f77c0e28fa Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Mon, 8 Jul 2024 15:42:08 +0100 Subject: [PATCH 15/24] try reduced period --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 4c005aa8302..3a61c6b57dc 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -4,7 +4,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { evaluation_periods = "1" metric_name = "HTTPCode_Target_5XX_Count" namespace = "AWS/ApplicationELB" - period = "300" + period = "60" statistic = "Sum" threshold = "1" alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" From a4216e0cd880bd1d731b7b1edb964ded07fa663e Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Mon, 8 Jul 2024 16:58:45 +0100 Subject: [PATCH 16/24] treat missing data as not breaching alarm status --- terraform/environments/cdpt-ifs/monitoring.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 3a61c6b57dc..a2771a79351 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -12,6 +12,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { dimensions = { LoadBalancer = "${local.application_name}-lb" } + treat_missing_data = "notBreaching" } resource "aws_sns_topic" "lb_5xx_alarm_topic" { From fe9c754ed2cb809dffbd765781972ee4d04556d8 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Tue, 9 Jul 2024 14:45:24 +0100 Subject: [PATCH 17/24] use full arn for lb dimension --- terraform/environments/cdpt-ifs/monitoring.tf | 16 ++++++++++------ terraform/environments/cdpt-ifs/nohup.out | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 terraform/environments/cdpt-ifs/nohup.out diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index a2771a79351..c4bae8e3a76 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -1,3 +1,11 @@ +data "aws_lb" "cdpt-ifs-lb" { + name = "cdpt-ifs-lb" +} + +resource "aws_sns_topic" "lb_5xx_alarm_topic" { + name = "lb_5xx_alarm_topic" +} + resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_name = "lb-5xx-errors" comparison_operator = "GreaterThanOrEqualToThreshold" @@ -10,13 +18,9 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { - LoadBalancer = "${local.application_name}-lb" + LoadBalancer = "data.aws_lb.cdpt-ifs-lb.arn" } - treat_missing_data = "notBreaching" -} - -resource "aws_sns_topic" "lb_5xx_alarm_topic" { - name = "lb_5xx_alarm_topic" + treat_missing_data = "missing" } # Pager duty integration diff --git a/terraform/environments/cdpt-ifs/nohup.out b/terraform/environments/cdpt-ifs/nohup.out new file mode 100644 index 00000000000..be6f93d296a --- /dev/null +++ b/terraform/environments/cdpt-ifs/nohup.out @@ -0,0 +1,3 @@ +nohup: ./ping500.sh: No such file or directory +nohup: ./ping500.sh: No such file or directory +nohup: ./ping500.sh: No such file or directory From b209e8776cec7d07741c34fa4923f6c78aafd59f Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Tue, 9 Jul 2024 14:51:37 +0100 Subject: [PATCH 18/24] remove quotes --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index c4bae8e3a76..b01d924dc7a 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -18,7 +18,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { - LoadBalancer = "data.aws_lb.cdpt-ifs-lb.arn" + LoadBalancer = data.aws_lb.cdpt-ifs-lb.arn } treat_missing_data = "missing" } From 301bb515734e1fad967a4467f99ff0c444ff1a59 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Tue, 9 Jul 2024 15:19:02 +0100 Subject: [PATCH 19/24] use correct lb arn --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index b01d924dc7a..92ded46cbda 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -18,7 +18,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { - LoadBalancer = data.aws_lb.cdpt-ifs-lb.arn + ResourceArn = module.lb_access_logs_enabled.load_balancer_arn } treat_missing_data = "missing" } From 6e3526ed9f73b29c55ba9d165d590860eb1a1323 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Wed, 10 Jul 2024 09:08:17 +0100 Subject: [PATCH 20/24] set missing data --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 92ded46cbda..b3fe6eff198 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -20,7 +20,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { dimensions = { ResourceArn = module.lb_access_logs_enabled.load_balancer_arn } - treat_missing_data = "missing" + treat_missing_data = "notBreaching" } # Pager duty integration From e2165ac13fe8394b15fd4376486acbe7055959af Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Wed, 10 Jul 2024 09:10:24 +0100 Subject: [PATCH 21/24] set period 300 --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index b3fe6eff198..1f0736dc971 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -12,7 +12,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { evaluation_periods = "1" metric_name = "HTTPCode_Target_5XX_Count" namespace = "AWS/ApplicationELB" - period = "60" + period = "300" statistic = "Sum" threshold = "1" alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" From 4c19fe31693b24f30a9c1b5708b65510e993795f Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Wed, 10 Jul 2024 09:42:28 +0100 Subject: [PATCH 22/24] hardcode lb resource to test --- terraform/environments/cdpt-ifs/monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 1f0736dc971..752c59db042 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -18,7 +18,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { - ResourceArn = module.lb_access_logs_enabled.load_balancer_arn + LoadBalancer = "app/cdpt-ifs-lb/57f7ca8467869532" } treat_missing_data = "notBreaching" } From dbce882bcba9c5b5ed94d8c0c3ca21a989649349 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Wed, 10 Jul 2024 11:48:11 +0100 Subject: [PATCH 23/24] dynamically construct lb resource to test --- terraform/environments/cdpt-ifs/monitoring.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/monitoring.tf b/terraform/environments/cdpt-ifs/monitoring.tf index 752c59db042..11e78ceb26d 100644 --- a/terraform/environments/cdpt-ifs/monitoring.tf +++ b/terraform/environments/cdpt-ifs/monitoring.tf @@ -6,6 +6,10 @@ resource "aws_sns_topic" "lb_5xx_alarm_topic" { name = "lb_5xx_alarm_topic" } +locals{ + lb_short_arn = join("/", slice(split("/", module.lb_access_logs_enabled.load_balancer_arn), 1, 4)) +} + resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_name = "lb-5xx-errors" comparison_operator = "GreaterThanOrEqualToThreshold" @@ -18,7 +22,7 @@ resource "aws_cloudwatch_metric_alarm" "lb_5xx_errors" { alarm_description = "This metric monitors 5xx errors on the targets behind the load balancer" alarm_actions = [aws_sns_topic.lb_5xx_alarm_topic.arn] dimensions = { - LoadBalancer = "app/cdpt-ifs-lb/57f7ca8467869532" + LoadBalancer = local.lb_short_arn } treat_missing_data = "notBreaching" } @@ -50,3 +54,6 @@ module "pagerduty_core_alerts" { pagerduty_integration_key = local.pagerduty_integration_keys["cdpt-ifs-alarms"] } +output "lb_short_arn" { + value = local.lb_short_arn +} From 43b63cfeff0df2f22317fbecef26949848639f86 Mon Sep 17 00:00:00 2001 From: Alistair Curtis Date: Wed, 10 Jul 2024 13:02:57 +0100 Subject: [PATCH 24/24] remove output file --- terraform/environments/cdpt-ifs/nohup.out | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 terraform/environments/cdpt-ifs/nohup.out diff --git a/terraform/environments/cdpt-ifs/nohup.out b/terraform/environments/cdpt-ifs/nohup.out deleted file mode 100644 index be6f93d296a..00000000000 --- a/terraform/environments/cdpt-ifs/nohup.out +++ /dev/null @@ -1,3 +0,0 @@ -nohup: ./ping500.sh: No such file or directory -nohup: ./ping500.sh: No such file or directory -nohup: ./ping500.sh: No such file or directory