From ca7c3c1678be8f03fbfc8a333f605c2d732df853 Mon Sep 17 00:00:00 2001 From: Graham Pengelly Date: Fri, 22 Apr 2016 11:20:04 +0100 Subject: [PATCH] Add email_alert_notifications project This project sets up an S3 bucket and a lambda function that parses files and renames them with a prefix (currently only `travel-advice-alerts`) and the `govuk_request_id` parsed from the email body. This allows us to more reliably verify that email alerts have been sent when a travel advice update has been published. An SES domain and RuleSet and an S3 event source are also required but can't currently be configured via Terraform. --- projects/email_alert_notifications/README.md | 1 + .../rename_email_files_with_request_id.py | 44 +++++++++++++++ .../rename_email_files_with_request_id.zip | Bin 0 -> 773 bytes .../resources/email_alert_notifications.tf | 53 ++++++++++++++++++ .../rename_email_files_with_request_id.zip | Bin 0 -> 773 bytes .../email_alert_s3_bucket_policy.json | 32 +++++++++++ .../templates/lambda_assume_role_policy.json | 13 +++++ .../put_and_delete_to_s3_policy.json | 15 +++++ .../templates/write_to_logs_policy.json | 14 +++++ 9 files changed, 172 insertions(+) create mode 100644 projects/email_alert_notifications/README.md create mode 100644 projects/email_alert_notifications/files/rename_email_files_with_request_id.py create mode 100644 projects/email_alert_notifications/files/rename_email_files_with_request_id.zip create mode 100644 projects/email_alert_notifications/resources/email_alert_notifications.tf create mode 100644 projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip create mode 100644 projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json create mode 100644 projects/email_alert_notifications/templates/lambda_assume_role_policy.json create mode 100644 projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json create mode 100644 projects/email_alert_notifications/templates/write_to_logs_policy.json diff --git a/projects/email_alert_notifications/README.md b/projects/email_alert_notifications/README.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/projects/email_alert_notifications/README.md @@ -0,0 +1 @@ + diff --git a/projects/email_alert_notifications/files/rename_email_files_with_request_id.py b/projects/email_alert_notifications/files/rename_email_files_with_request_id.py new file mode 100644 index 0000000..fe1669e --- /dev/null +++ b/projects/email_alert_notifications/files/rename_email_files_with_request_id.py @@ -0,0 +1,44 @@ +import boto3 +import urllib +import re +import uuid +from botocore.exceptions import ClientError + +S3 = boto3.client('s3') + +REQUEST_ID_REGEX = re.compile(r'data-govuk-request-id=(?:3D){,1}"([0-9\-\.]+)"') + +def lambda_handler(event, context): + bucket_name = source_bucket_name(event) + key = source_key(event) + request_id = parse_request_id(bucket_name, key) + prefix = file_prefix(event, request_id) + move_file(bucket_name, key, request_id, prefix) + +def source_bucket_name(event): + return event['Records'][0]['s3']['bucket']['name'] + +def source_key(event): + return urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8') + +def parse_request_id(bucket_name, key): + m = REQUEST_ID_REGEX.search(email_body(bucket_name, key)) + if m: + return m.group(1) + +def email_body(bucket_name, key): + response = S3.get_object(Bucket=bucket_name, Key=key) + return response["Body"].read() + +def move_file(bucket_name, key, request_id, prefix): + S3.copy_object( + Bucket=bucket_name, + CopySource='%s/%s' % (bucket_name, key), + Key='%s/%s.msg' % (prefix, request_id or uuid.uuid4())) + S3.delete_object(Bucket=bucket_name, Key=key) + +def file_prefix(event, request_id): + if request_id: + return "travel-advice-alerts" + else: + return "no-request-id" diff --git a/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip new file mode 100644 index 0000000000000000000000000000000000000000..54e7df4ac70092c01942f77a0352298cc988fc07 GIT binary patch literal 773 zcmWIWW@Zs#U|`^2&|NmoW6j6id8$kd47#ig45|z=3`MDViMgrqskw=nIq_+kIjP0* z<(VZJ@kObHrK!aw@tG-l1(l&8oD9rv4N~C`I;6r&E4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;@sJLB9Y1 literal 0 HcmV?d00001 diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf new file mode 100644 index 0000000..02e4347 --- /dev/null +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -0,0 +1,53 @@ +variable "s3_bucket_name" { + default = "govuk-email-alert-notifications" +} + +resource "template_file" "s3_bucket_policy" { + template = "${file("templates/email_alert_s3_bucket_policy.json")}" + vars { + account_id = "${element(split(":", aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn), 4)}" + bucket_name = "${var.s3_bucket_name}" + lambda_role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" + } +} + +resource "template_file" "put_and_delete_to_email_alert_bucket_policy" { + template = "${file("templates/put_and_delete_to_s3_policy.json")}" + vars { + resource_arn = "${aws_s3_bucket.email_alert_inbox_bucket.arn}" + } +} + +resource "aws_s3_bucket" "email_alert_inbox_bucket" { + bucket = "${var.s3_bucket_name}" + acl = "public-read" + policy = "${template_file.s3_bucket_policy.rendered}" +} + +resource "aws_iam_role" "lambda_execute_and_write_to_email_alert_bucket" { + name = "lambda_execute_and_write_to_email_alert_bucket" + assume_role_policy = "${file("templates/lambda_assume_role_policy.json")}" +} + +resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { + name = "put_and_delete_to_email_alert_bucket" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" + policy = "${template_file.put_and_delete_to_email_alert_bucket_policy.rendered}" +} + +resource "aws_iam_role_policy" "write_to_logs" { + name = "write_to_logs" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" + policy = "${file("templates/write_to_logs_policy.json")}" +} + +resource "aws_lambda_function" "rename_email_files_with_request_id"{ + filename = "rename_email_files_with_request_id.zip" + function_name = "rename_email_files_with_request_id" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" + handler = "rename_email_files_with_request_id.lambda_handler" + runtime = "python2.7" + source_code_hash = "${base64sha256(file("rename_email_files_with_request_id.zip"))}" +} + + diff --git a/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip new file mode 100644 index 0000000000000000000000000000000000000000..d371ba800e6c16fa86cfd249492193393a460085 GIT binary patch literal 773 zcmWIWW@Zs#U|`^2n6rAS$C{73^HiA_7<5?~7*rW#7>ZK!5_41IQ*#qDbK=u7b5e`r z%QH(d;)_xXOH+$W;xkk93MxZGI2o8ZEF{Cjtt7)sE4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;>9aK)(P0 literal 0 HcmV?d00001 diff --git a/projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json b/projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json new file mode 100644 index 0000000..357eec6 --- /dev/null +++ b/projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json @@ -0,0 +1,32 @@ +{ + "Version": "2012-10-17", + "Id": "${uuid()}", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "${lambda_role}" + }, + "Action": [ + "s3:PutObject", + "s3:DeleteObject", + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::${bucket_name}/*" + }, + { + "Sid": "GiveSESPermissionToWriteEmail", + "Effect": "Allow", + "Principal": { + "Service": "ses.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::${bucket_name}/*", + "Condition": { + "StringEquals": { + "aws:Referer": "${account_id}" + } + } + } + ] +} diff --git a/projects/email_alert_notifications/templates/lambda_assume_role_policy.json b/projects/email_alert_notifications/templates/lambda_assume_role_policy.json new file mode 100644 index 0000000..7dee58b --- /dev/null +++ b/projects/email_alert_notifications/templates/lambda_assume_role_policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} diff --git a/projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json b/projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json new file mode 100644 index 0000000..125e0dc --- /dev/null +++ b/projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:getObject", + "s3:putObject", + "s3:deleteObject", + "s3:putObjectAcl" + ], + "Effect": "Allow", + "Resource": "${resource_arn}" + } + ] +} diff --git a/projects/email_alert_notifications/templates/write_to_logs_policy.json b/projects/email_alert_notifications/templates/write_to_logs_policy.json new file mode 100644 index 0000000..63a7264 --- /dev/null +++ b/projects/email_alert_notifications/templates/write_to_logs_policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] +}