From a0be1dd292ac814dd696b401babb9a0181a69c21 Mon Sep 17 00:00:00 2001 From: Graham Pengelly Date: Fri, 22 Apr 2016 11:20:04 +0100 Subject: [PATCH 1/7] Add email_alert_notifications project This project sets up an S3 bucket and a lambda function that parses files and renames them with a prefix (currently only `travel-advice-alerts`) and the `govuk_request_id` parsed from the email body. This allows us to more reliably verify that email alerts have been sent when a travel advice update has been published. An SES domain and ruleset are also required but can't currently be configured via Terraform. --- projects/email_alert_notifications/README.md | 1 + .../rename_email_files_with_request_id.zip | Bin 0 -> 773 bytes .../resources/email_alert_notifications.tf | 53 ++++++++++++++++++ .../resources/files/.gitignore | 1 + .../rename_email_files_with_request_id.py | 44 +++++++++++++++ .../rename_email_files_with_request_id.zip | Bin 0 -> 773 bytes .../email_alert_s3_bucket_policy.json | 32 +++++++++++ .../templates/lambda_assume_role_policy.json | 13 +++++ .../put_and_delete_to_s3_policy.json | 15 +++++ .../templates/write_to_logs_policy.json | 14 +++++ 10 files changed, 173 insertions(+) create mode 100644 projects/email_alert_notifications/README.md create mode 100644 projects/email_alert_notifications/files/rename_email_files_with_request_id.zip create mode 100644 projects/email_alert_notifications/resources/email_alert_notifications.tf create mode 100644 projects/email_alert_notifications/resources/files/.gitignore create mode 100644 projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py create mode 100644 projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip create mode 100644 projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json create mode 100644 projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json create mode 100644 projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json create mode 100644 projects/email_alert_notifications/resources/templates/write_to_logs_policy.json diff --git a/projects/email_alert_notifications/README.md b/projects/email_alert_notifications/README.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/projects/email_alert_notifications/README.md @@ -0,0 +1 @@ + diff --git a/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip new file mode 100644 index 0000000000000000000000000000000000000000..54e7df4ac70092c01942f77a0352298cc988fc07 GIT binary patch literal 773 zcmWIWW@Zs#U|`^2&|NmoW6j6id8$kd47#ig45|z=3`MDViMgrqskw=nIq_+kIjP0* z<(VZJ@kObHrK!aw@tG-l1(l&8oD9rv4N~C`I;6r&E4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;@sJLB9Y1 literal 0 HcmV?d00001 diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf new file mode 100644 index 0000000..02e4347 --- /dev/null +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -0,0 +1,53 @@ +variable "s3_bucket_name" { + default = "govuk-email-alert-notifications" +} + +resource "template_file" "s3_bucket_policy" { + template = "${file("templates/email_alert_s3_bucket_policy.json")}" + vars { + account_id = "${element(split(":", aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn), 4)}" + bucket_name = "${var.s3_bucket_name}" + lambda_role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" + } +} + +resource "template_file" "put_and_delete_to_email_alert_bucket_policy" { + template = "${file("templates/put_and_delete_to_s3_policy.json")}" + vars { + resource_arn = "${aws_s3_bucket.email_alert_inbox_bucket.arn}" + } +} + +resource "aws_s3_bucket" "email_alert_inbox_bucket" { + bucket = "${var.s3_bucket_name}" + acl = "public-read" + policy = "${template_file.s3_bucket_policy.rendered}" +} + +resource "aws_iam_role" "lambda_execute_and_write_to_email_alert_bucket" { + name = "lambda_execute_and_write_to_email_alert_bucket" + assume_role_policy = "${file("templates/lambda_assume_role_policy.json")}" +} + +resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { + name = "put_and_delete_to_email_alert_bucket" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" + policy = "${template_file.put_and_delete_to_email_alert_bucket_policy.rendered}" +} + +resource "aws_iam_role_policy" "write_to_logs" { + name = "write_to_logs" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" + policy = "${file("templates/write_to_logs_policy.json")}" +} + +resource "aws_lambda_function" "rename_email_files_with_request_id"{ + filename = "rename_email_files_with_request_id.zip" + function_name = "rename_email_files_with_request_id" + role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" + handler = "rename_email_files_with_request_id.lambda_handler" + runtime = "python2.7" + source_code_hash = "${base64sha256(file("rename_email_files_with_request_id.zip"))}" +} + + diff --git a/projects/email_alert_notifications/resources/files/.gitignore b/projects/email_alert_notifications/resources/files/.gitignore new file mode 100644 index 0000000..c4c4ffc --- /dev/null +++ b/projects/email_alert_notifications/resources/files/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py b/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py new file mode 100644 index 0000000..fe1669e --- /dev/null +++ b/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py @@ -0,0 +1,44 @@ +import boto3 +import urllib +import re +import uuid +from botocore.exceptions import ClientError + +S3 = boto3.client('s3') + +REQUEST_ID_REGEX = re.compile(r'data-govuk-request-id=(?:3D){,1}"([0-9\-\.]+)"') + +def lambda_handler(event, context): + bucket_name = source_bucket_name(event) + key = source_key(event) + request_id = parse_request_id(bucket_name, key) + prefix = file_prefix(event, request_id) + move_file(bucket_name, key, request_id, prefix) + +def source_bucket_name(event): + return event['Records'][0]['s3']['bucket']['name'] + +def source_key(event): + return urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8') + +def parse_request_id(bucket_name, key): + m = REQUEST_ID_REGEX.search(email_body(bucket_name, key)) + if m: + return m.group(1) + +def email_body(bucket_name, key): + response = S3.get_object(Bucket=bucket_name, Key=key) + return response["Body"].read() + +def move_file(bucket_name, key, request_id, prefix): + S3.copy_object( + Bucket=bucket_name, + CopySource='%s/%s' % (bucket_name, key), + Key='%s/%s.msg' % (prefix, request_id or uuid.uuid4())) + S3.delete_object(Bucket=bucket_name, Key=key) + +def file_prefix(event, request_id): + if request_id: + return "travel-advice-alerts" + else: + return "no-request-id" diff --git a/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip new file mode 100644 index 0000000000000000000000000000000000000000..d371ba800e6c16fa86cfd249492193393a460085 GIT binary patch literal 773 zcmWIWW@Zs#U|`^2n6rAS$C{73^HiA_7<5?~7*rW#7>ZK!5_41IQ*#qDbK=u7b5e`r z%QH(d;)_xXOH+$W;xkk93MxZGI2o8ZEF{Cjtt7)sE4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;>9aK)(P0 literal 0 HcmV?d00001 diff --git a/projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json b/projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json new file mode 100644 index 0000000..357eec6 --- /dev/null +++ b/projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json @@ -0,0 +1,32 @@ +{ + "Version": "2012-10-17", + "Id": "${uuid()}", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "${lambda_role}" + }, + "Action": [ + "s3:PutObject", + "s3:DeleteObject", + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::${bucket_name}/*" + }, + { + "Sid": "GiveSESPermissionToWriteEmail", + "Effect": "Allow", + "Principal": { + "Service": "ses.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::${bucket_name}/*", + "Condition": { + "StringEquals": { + "aws:Referer": "${account_id}" + } + } + } + ] +} diff --git a/projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json b/projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json new file mode 100644 index 0000000..7dee58b --- /dev/null +++ b/projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} diff --git a/projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json b/projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json new file mode 100644 index 0000000..125e0dc --- /dev/null +++ b/projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:getObject", + "s3:putObject", + "s3:deleteObject", + "s3:putObjectAcl" + ], + "Effect": "Allow", + "Resource": "${resource_arn}" + } + ] +} diff --git a/projects/email_alert_notifications/resources/templates/write_to_logs_policy.json b/projects/email_alert_notifications/resources/templates/write_to_logs_policy.json new file mode 100644 index 0000000..63a7264 --- /dev/null +++ b/projects/email_alert_notifications/resources/templates/write_to_logs_policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] +} From 523621d831d42e68ecda93f6c9283731fda61032 Mon Sep 17 00:00:00 2001 From: Graham Pengelly Date: Mon, 25 Apr 2016 21:42:35 +0100 Subject: [PATCH 2/7] Add s3 bucket notification This configures PUT events to the s3 bucket to invoke the lambda. NB this is only supported in terraform v0.6.15 --- .../email_alert_notifications/files/.gitignore | 1 + .../resources/email_alert_notifications.tf | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 projects/email_alert_notifications/files/.gitignore diff --git a/projects/email_alert_notifications/files/.gitignore b/projects/email_alert_notifications/files/.gitignore new file mode 100644 index 0000000..c4c4ffc --- /dev/null +++ b/projects/email_alert_notifications/files/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf index 02e4347..edd3446 100644 --- a/projects/email_alert_notifications/resources/email_alert_notifications.tf +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -50,4 +50,18 @@ resource "aws_lambda_function" "rename_email_files_with_request_id"{ source_code_hash = "${base64sha256(file("rename_email_files_with_request_id.zip"))}" } +resource "aws_lambda_permission" "allow_email_alert_inbox_bucket" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = "${aws_lambda_function.rename_email_files_with_request_id.arn}" + principal = "s3.amazonaws.com" + source_arn = "${aws_s3_bucket.email_alert_inbox_bucket.arn}" +} +resource "aws_s3_bucket_notification" "email_alert_inbox_bucket_notification" { + bucket = "${aws_s3_bucket.email_alert_inbox_bucket.id}" + lambda_function { + lambda_function_arn = "${aws_lambda_function.rename_email_files_with_request_id.arn}" + events = ["s3:ObjectCreated:Put"] + } +} From bd1ef1cf6772a11863560f5de05c607c971f4805 Mon Sep 17 00:00:00 2001 From: Graham Pengelly Date: Fri, 22 Apr 2016 13:02:52 +0100 Subject: [PATCH 3/7] Update README --- projects/email_alert_notifications/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/projects/email_alert_notifications/README.md b/projects/email_alert_notifications/README.md index 8b13789..db0969a 100644 --- a/projects/email_alert_notifications/README.md +++ b/projects/email_alert_notifications/README.md @@ -1 +1,20 @@ +# Email Alert Notifications + +This project sets up an S3 bucket and a Lambda that parses files, +extracts `data-govuk-request-id` and renames the file accordingly as +either: + + travel-advice-alerts/.msg + +or if the file doesn't contain a `data-govuk-request-id` + + no-request-id/.msg + +## Manual Steps + +There is a manual step required for this 'app'. + +* An SES domain with a RuleSet containing an action that writes to the S3 +bucket (`govuk-email-alert-notifications`). An address on this domain will be +subscribed to all of the travel advice country alerts From 2d35845d8433661e03732c7cc21edb32cde4f1fc Mon Sep 17 00:00:00 2001 From: Graham Pengelly Date: Thu, 19 May 2016 14:37:15 +0100 Subject: [PATCH 4/7] Get lambda from S3 This commit moves the lambda source code out of this repo and now pulls it from S3 (`govuk-lambda-applications-`) The lambda zip file version needs to be passed in as an environment variable `TF_VAR_rename_email_files_with_request_id_version` --- .../files/.gitignore | 1 - .../rename_email_files_with_request_id.zip | Bin 773 -> 0 bytes .../resources/email_alert_notifications.tf | 21 +++++++-- .../resources/files/.gitignore | 1 - .../rename_email_files_with_request_id.py | 44 ------------------ .../rename_email_files_with_request_id.zip | Bin 773 -> 0 bytes 6 files changed, 16 insertions(+), 51 deletions(-) delete mode 100644 projects/email_alert_notifications/files/.gitignore delete mode 100644 projects/email_alert_notifications/files/rename_email_files_with_request_id.zip delete mode 100644 projects/email_alert_notifications/resources/files/.gitignore delete mode 100644 projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py delete mode 100644 projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip diff --git a/projects/email_alert_notifications/files/.gitignore b/projects/email_alert_notifications/files/.gitignore deleted file mode 100644 index c4c4ffc..0000000 --- a/projects/email_alert_notifications/files/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.zip diff --git a/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/files/rename_email_files_with_request_id.zip deleted file mode 100644 index 54e7df4ac70092c01942f77a0352298cc988fc07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 773 zcmWIWW@Zs#U|`^2&|NmoW6j6id8$kd47#ig45|z=3`MDViMgrqskw=nIq_+kIjP0* z<(VZJ@kObHrK!aw@tG-l1(l&8oD9rv4N~C`I;6r&E4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;@sJLB9Y1 diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf index edd3446..a792d45 100644 --- a/projects/email_alert_notifications/resources/email_alert_notifications.tf +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -2,11 +2,21 @@ variable "s3_bucket_name" { default = "govuk-email-alert-notifications" } +variable "environment"{ +} + +variable "rename_email_files_with_request_id_version"{ +} + +variable "lambda_bucket"{ + default = "govuk-lambda-applications" +} + resource "template_file" "s3_bucket_policy" { template = "${file("templates/email_alert_s3_bucket_policy.json")}" vars { account_id = "${element(split(":", aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn), 4)}" - bucket_name = "${var.s3_bucket_name}" + bucket_name = "${var.s3_bucket_name}-${var.environment}" lambda_role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" } } @@ -19,13 +29,13 @@ resource "template_file" "put_and_delete_to_email_alert_bucket_policy" { } resource "aws_s3_bucket" "email_alert_inbox_bucket" { - bucket = "${var.s3_bucket_name}" + bucket = "${var.s3_bucket_name}-${var.environment}" acl = "public-read" policy = "${template_file.s3_bucket_policy.rendered}" } resource "aws_iam_role" "lambda_execute_and_write_to_email_alert_bucket" { - name = "lambda_execute_and_write_to_email_alert_bucket" + name = "lambda_execute_and_write_to_email_alert_bucket" assume_role_policy = "${file("templates/lambda_assume_role_policy.json")}" } @@ -42,12 +52,13 @@ resource "aws_iam_role_policy" "write_to_logs" { } resource "aws_lambda_function" "rename_email_files_with_request_id"{ - filename = "rename_email_files_with_request_id.zip" + s3_bucket = "${var.lambda_bucket}-${var.environment}" + s3_key="rename_email_files_with_request_id.zip" + s3_object_version="${var.rename_email_files_with_request_id_version}" function_name = "rename_email_files_with_request_id" role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" handler = "rename_email_files_with_request_id.lambda_handler" runtime = "python2.7" - source_code_hash = "${base64sha256(file("rename_email_files_with_request_id.zip"))}" } resource "aws_lambda_permission" "allow_email_alert_inbox_bucket" { diff --git a/projects/email_alert_notifications/resources/files/.gitignore b/projects/email_alert_notifications/resources/files/.gitignore deleted file mode 100644 index c4c4ffc..0000000 --- a/projects/email_alert_notifications/resources/files/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.zip diff --git a/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py b/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py deleted file mode 100644 index fe1669e..0000000 --- a/projects/email_alert_notifications/resources/files/rename_email_files_with_request_id.py +++ /dev/null @@ -1,44 +0,0 @@ -import boto3 -import urllib -import re -import uuid -from botocore.exceptions import ClientError - -S3 = boto3.client('s3') - -REQUEST_ID_REGEX = re.compile(r'data-govuk-request-id=(?:3D){,1}"([0-9\-\.]+)"') - -def lambda_handler(event, context): - bucket_name = source_bucket_name(event) - key = source_key(event) - request_id = parse_request_id(bucket_name, key) - prefix = file_prefix(event, request_id) - move_file(bucket_name, key, request_id, prefix) - -def source_bucket_name(event): - return event['Records'][0]['s3']['bucket']['name'] - -def source_key(event): - return urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8') - -def parse_request_id(bucket_name, key): - m = REQUEST_ID_REGEX.search(email_body(bucket_name, key)) - if m: - return m.group(1) - -def email_body(bucket_name, key): - response = S3.get_object(Bucket=bucket_name, Key=key) - return response["Body"].read() - -def move_file(bucket_name, key, request_id, prefix): - S3.copy_object( - Bucket=bucket_name, - CopySource='%s/%s' % (bucket_name, key), - Key='%s/%s.msg' % (prefix, request_id or uuid.uuid4())) - S3.delete_object(Bucket=bucket_name, Key=key) - -def file_prefix(event, request_id): - if request_id: - return "travel-advice-alerts" - else: - return "no-request-id" diff --git a/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip b/projects/email_alert_notifications/resources/rename_email_files_with_request_id.zip deleted file mode 100644 index d371ba800e6c16fa86cfd249492193393a460085..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 773 zcmWIWW@Zs#U|`^2n6rAS$C{73^HiA_7<5?~7*rW#7>ZK!5_41IQ*#qDbK=u7b5e`r z%QH(d;)_xXOH+$W;xkk93MxZGI2o8ZEF{Cjtt7)sE4UdLS-vtdFo2Dn8WP(*+d-uE zdw7R{Oy-u|OZamYM9R#kTw1QCE3!#xaq(Hd#eLTDiVNS=zrT7hW0qO&XN8%sEuYW( z{qVykv7p^6FC;xHs5aQ7QE_X&%A7}S#_H1-u4m!EX?)8S9jhXQnkQ_pfMTuKkQT%i3eP28)( zY(dG1n@xM>pOOOa4?MRBW|eZC&sB@!^xR!~Dze^SVX$Evr#&3K-dS?cz5%Ed6Q;K{eooid%dkAKxM z{RKCdf7Q9Abm*GLyQn{pdcJY?bzgQW(X4LlEpN}7=^5v)h&y^y)UnmQI-DqC4 z>Z5s|-}`qHH{_}B{hI2*r0^{QB6^o-x3i xkx7mjS1ObMrcMS1V1i;;(g5wX%a0x26(fwfec{;!fYVj0!-`-3;>9aK)(P0 From d612f2d1474716d0dd1946771c32ab44395b4ce5 Mon Sep 17 00:00:00 2001 From: Laura Martin Date: Tue, 24 May 2016 13:37:03 +0100 Subject: [PATCH 5/7] Pull object versionid There was a requirement to be able to pull in the latest VersionID of a specific object in a bucket, with a prerequisite that it was triggered by a Jenkins deployment job. This changes seeks to add that functionality by allowing the addition of an environmental variable in the deploy job named "TF_VAR_LAMBDA_FILENAME". I updated this in the project it's used so it could be used more generically in the future. There might be a bit of extra work if we wish to override the VersionID for whatever reason. --- Rakefile | 22 +++++++++++++++++-- .../resources/email_alert_notifications.tf | 11 ++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Rakefile b/Rakefile index 759a049..187c3b3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,7 @@ require 'fileutils' require 'rspec/core/rake_task' require 'tmpdir' +require 'aws-sdk-resources' PROJECT_DIR = 'projects' @@ -110,7 +111,7 @@ end desc 'Apply the terraform resources' -task apply: [:configure_state] do +task apply: [:configure_state, :latest_version_id] do tmp_dir = _flatten_project puts "terraform apply -var-file=variables/#{deploy_env}.tfvars #{tmp_dir}" @@ -121,6 +122,23 @@ task apply: [:configure_state] do end +desc 'Latest Lambda version ID' +task :latest_version_id do + lambda_filename = ENV['TF_VAR_LAMBDA_FILENAME'] + target_file = "#{project_name}/#{lambda_filename}" + if lambda_filename + s3 = Aws::S3::Resource.new(region: 'eu-west-1') + bucket = s3.bucket("govuk-lambda-applications-#{deploy_env}") + begin + version_id = bucket.object(target_file).version_id + rescue Aws::S3::Errors::NotFound + abort "File not found: s3://govuk-lambda-applications-#{deploy_env}/#{target_file}" + end + ENV['TF_VAR_LAMBDA_VERSIONID'] = version_id + end +end + + desc 'Destroy the terraform resources' task destroy: [:configure_state] do tmp_dir = _flatten_project @@ -134,7 +152,7 @@ end desc 'Show the plan' -task plan: [:configure_state] do +task plan: [:configure_state, :latest_version_id] do tmp_dir = _flatten_project system("terraform plan -module-depth=-1 -var-file=variables/#{deploy_env}.tfvars #{tmp_dir}") diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf index a792d45..86e3343 100644 --- a/projects/email_alert_notifications/resources/email_alert_notifications.tf +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -5,7 +5,10 @@ variable "s3_bucket_name" { variable "environment"{ } -variable "rename_email_files_with_request_id_version"{ +variable "LAMBDA_FILENAME"{ +} + +variable "LAMBDA_VERSIONID"{ } variable "lambda_bucket"{ @@ -48,13 +51,13 @@ resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { resource "aws_iam_role_policy" "write_to_logs" { name = "write_to_logs" role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" - policy = "${file("templates/write_to_logs_policy.json")}" + policy = "${file("templates/email_alert_notifications/write_to_logs_policy.json")}" } resource "aws_lambda_function" "rename_email_files_with_request_id"{ s3_bucket = "${var.lambda_bucket}-${var.environment}" - s3_key="rename_email_files_with_request_id.zip" - s3_object_version="${var.rename_email_files_with_request_id_version}" + s3_key="${var.LAMBDA_FILENAME}" + s3_object_version="${var.LAMBDA_VERSIONID}" function_name = "rename_email_files_with_request_id" role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn}" handler = "rename_email_files_with_request_id.lambda_handler" From 48ad5776566331d07fea4a942a986bc2982b7eb1 Mon Sep 17 00:00:00 2001 From: Laura Martin Date: Tue, 24 May 2016 13:41:16 +0100 Subject: [PATCH 6/7] Update location of templates Terraform only works with stuff when it is quoted from the relative directory. Our rake task pushes everything into a temporary directory and then feeds that in during the plan/apply system call. In the Terraform code we specified a relative path, so to fix this I have created a new templates directory with the project name underneath it. Another solution would be to pass in the "tmpdir" variable in the Rake task as env var, but I thought this was not as tidy as just having a separate directory especially for templates. --- .../resources/email_alert_notifications.tf | 6 ++-- .../email_alert_s3_bucket_policy.json | 32 +++++++++++++++++++ .../lambda_assume_role_policy.json | 13 ++++++++ .../put_and_delete_to_s3_policy.json | 15 +++++++++ .../write_to_logs_policy.json | 14 ++++++++ 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 templates/email_alert_notifications/email_alert_s3_bucket_policy.json create mode 100644 templates/email_alert_notifications/lambda_assume_role_policy.json create mode 100644 templates/email_alert_notifications/put_and_delete_to_s3_policy.json create mode 100644 templates/email_alert_notifications/write_to_logs_policy.json diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf index 86e3343..7a9422d 100644 --- a/projects/email_alert_notifications/resources/email_alert_notifications.tf +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -16,7 +16,7 @@ variable "lambda_bucket"{ } resource "template_file" "s3_bucket_policy" { - template = "${file("templates/email_alert_s3_bucket_policy.json")}" + template = "${file("templates/email_alert_notifications/email_alert_s3_bucket_policy.json")}" vars { account_id = "${element(split(":", aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn), 4)}" bucket_name = "${var.s3_bucket_name}-${var.environment}" @@ -25,7 +25,7 @@ resource "template_file" "s3_bucket_policy" { } resource "template_file" "put_and_delete_to_email_alert_bucket_policy" { - template = "${file("templates/put_and_delete_to_s3_policy.json")}" + template = "${file("templates/email_alert_notifications/put_and_delete_to_s3_policy.json")}" vars { resource_arn = "${aws_s3_bucket.email_alert_inbox_bucket.arn}" } @@ -39,7 +39,7 @@ resource "aws_s3_bucket" "email_alert_inbox_bucket" { resource "aws_iam_role" "lambda_execute_and_write_to_email_alert_bucket" { name = "lambda_execute_and_write_to_email_alert_bucket" - assume_role_policy = "${file("templates/lambda_assume_role_policy.json")}" + assume_role_policy = "${file("templates/email_alert_notifications/lambda_assume_role_policy.json")}" } resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { diff --git a/templates/email_alert_notifications/email_alert_s3_bucket_policy.json b/templates/email_alert_notifications/email_alert_s3_bucket_policy.json new file mode 100644 index 0000000..357eec6 --- /dev/null +++ b/templates/email_alert_notifications/email_alert_s3_bucket_policy.json @@ -0,0 +1,32 @@ +{ + "Version": "2012-10-17", + "Id": "${uuid()}", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "${lambda_role}" + }, + "Action": [ + "s3:PutObject", + "s3:DeleteObject", + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::${bucket_name}/*" + }, + { + "Sid": "GiveSESPermissionToWriteEmail", + "Effect": "Allow", + "Principal": { + "Service": "ses.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::${bucket_name}/*", + "Condition": { + "StringEquals": { + "aws:Referer": "${account_id}" + } + } + } + ] +} diff --git a/templates/email_alert_notifications/lambda_assume_role_policy.json b/templates/email_alert_notifications/lambda_assume_role_policy.json new file mode 100644 index 0000000..7dee58b --- /dev/null +++ b/templates/email_alert_notifications/lambda_assume_role_policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} diff --git a/templates/email_alert_notifications/put_and_delete_to_s3_policy.json b/templates/email_alert_notifications/put_and_delete_to_s3_policy.json new file mode 100644 index 0000000..125e0dc --- /dev/null +++ b/templates/email_alert_notifications/put_and_delete_to_s3_policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:getObject", + "s3:putObject", + "s3:deleteObject", + "s3:putObjectAcl" + ], + "Effect": "Allow", + "Resource": "${resource_arn}" + } + ] +} diff --git a/templates/email_alert_notifications/write_to_logs_policy.json b/templates/email_alert_notifications/write_to_logs_policy.json new file mode 100644 index 0000000..63a7264 --- /dev/null +++ b/templates/email_alert_notifications/write_to_logs_policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] +} From 1cfb37562da8dd8a83404ba479d1c41c2c9de01b Mon Sep 17 00:00:00 2001 From: Alex Muller Date: Wed, 1 Jun 2016 15:42:11 +0100 Subject: [PATCH 7/7] Move templates into project directory And update paths. This keeps everything together in the same directory. --- .../resources/email_alert_notifications.tf | 8 ++--- .../email_alert_s3_bucket_policy.json | 0 .../templates/lambda_assume_role_policy.json | 0 .../put_and_delete_to_s3_policy.json | 0 .../templates/write_to_logs_policy.json | 0 .../email_alert_s3_bucket_policy.json | 32 ------------------- .../lambda_assume_role_policy.json | 13 -------- .../put_and_delete_to_s3_policy.json | 15 --------- .../write_to_logs_policy.json | 14 -------- 9 files changed, 4 insertions(+), 78 deletions(-) rename projects/email_alert_notifications/{resources => }/templates/email_alert_s3_bucket_policy.json (100%) rename projects/email_alert_notifications/{resources => }/templates/lambda_assume_role_policy.json (100%) rename projects/email_alert_notifications/{resources => }/templates/put_and_delete_to_s3_policy.json (100%) rename projects/email_alert_notifications/{resources => }/templates/write_to_logs_policy.json (100%) delete mode 100644 templates/email_alert_notifications/email_alert_s3_bucket_policy.json delete mode 100644 templates/email_alert_notifications/lambda_assume_role_policy.json delete mode 100644 templates/email_alert_notifications/put_and_delete_to_s3_policy.json delete mode 100644 templates/email_alert_notifications/write_to_logs_policy.json diff --git a/projects/email_alert_notifications/resources/email_alert_notifications.tf b/projects/email_alert_notifications/resources/email_alert_notifications.tf index 7a9422d..9807284 100644 --- a/projects/email_alert_notifications/resources/email_alert_notifications.tf +++ b/projects/email_alert_notifications/resources/email_alert_notifications.tf @@ -16,7 +16,7 @@ variable "lambda_bucket"{ } resource "template_file" "s3_bucket_policy" { - template = "${file("templates/email_alert_notifications/email_alert_s3_bucket_policy.json")}" + template = "${file("projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json")}" vars { account_id = "${element(split(":", aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.arn), 4)}" bucket_name = "${var.s3_bucket_name}-${var.environment}" @@ -25,7 +25,7 @@ resource "template_file" "s3_bucket_policy" { } resource "template_file" "put_and_delete_to_email_alert_bucket_policy" { - template = "${file("templates/email_alert_notifications/put_and_delete_to_s3_policy.json")}" + template = "${file("projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json")}" vars { resource_arn = "${aws_s3_bucket.email_alert_inbox_bucket.arn}" } @@ -39,7 +39,7 @@ resource "aws_s3_bucket" "email_alert_inbox_bucket" { resource "aws_iam_role" "lambda_execute_and_write_to_email_alert_bucket" { name = "lambda_execute_and_write_to_email_alert_bucket" - assume_role_policy = "${file("templates/email_alert_notifications/lambda_assume_role_policy.json")}" + assume_role_policy = "${file("projects/email_alert_notifications/templates/lambda_assume_role_policy.json")}" } resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { @@ -51,7 +51,7 @@ resource "aws_iam_role_policy" "put_and_delete_to_email_alert_bucket" { resource "aws_iam_role_policy" "write_to_logs" { name = "write_to_logs" role = "${aws_iam_role.lambda_execute_and_write_to_email_alert_bucket.id}" - policy = "${file("templates/email_alert_notifications/write_to_logs_policy.json")}" + policy = "${file("projects/email_alert_notifications/templates/write_to_logs_policy.json")}" } resource "aws_lambda_function" "rename_email_files_with_request_id"{ diff --git a/projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json b/projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json similarity index 100% rename from projects/email_alert_notifications/resources/templates/email_alert_s3_bucket_policy.json rename to projects/email_alert_notifications/templates/email_alert_s3_bucket_policy.json diff --git a/projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json b/projects/email_alert_notifications/templates/lambda_assume_role_policy.json similarity index 100% rename from projects/email_alert_notifications/resources/templates/lambda_assume_role_policy.json rename to projects/email_alert_notifications/templates/lambda_assume_role_policy.json diff --git a/projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json b/projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json similarity index 100% rename from projects/email_alert_notifications/resources/templates/put_and_delete_to_s3_policy.json rename to projects/email_alert_notifications/templates/put_and_delete_to_s3_policy.json diff --git a/projects/email_alert_notifications/resources/templates/write_to_logs_policy.json b/projects/email_alert_notifications/templates/write_to_logs_policy.json similarity index 100% rename from projects/email_alert_notifications/resources/templates/write_to_logs_policy.json rename to projects/email_alert_notifications/templates/write_to_logs_policy.json diff --git a/templates/email_alert_notifications/email_alert_s3_bucket_policy.json b/templates/email_alert_notifications/email_alert_s3_bucket_policy.json deleted file mode 100644 index 357eec6..0000000 --- a/templates/email_alert_notifications/email_alert_s3_bucket_policy.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "Version": "2012-10-17", - "Id": "${uuid()}", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "${lambda_role}" - }, - "Action": [ - "s3:PutObject", - "s3:DeleteObject", - "s3:GetObject" - ], - "Resource": "arn:aws:s3:::${bucket_name}/*" - }, - { - "Sid": "GiveSESPermissionToWriteEmail", - "Effect": "Allow", - "Principal": { - "Service": "ses.amazonaws.com" - }, - "Action": "s3:PutObject", - "Resource": "arn:aws:s3:::${bucket_name}/*", - "Condition": { - "StringEquals": { - "aws:Referer": "${account_id}" - } - } - } - ] -} diff --git a/templates/email_alert_notifications/lambda_assume_role_policy.json b/templates/email_alert_notifications/lambda_assume_role_policy.json deleted file mode 100644 index 7dee58b..0000000 --- a/templates/email_alert_notifications/lambda_assume_role_policy.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Principal": { - "Service": "lambda.amazonaws.com" - }, - "Effect": "Allow", - "Sid": "" - } - ] -} diff --git a/templates/email_alert_notifications/put_and_delete_to_s3_policy.json b/templates/email_alert_notifications/put_and_delete_to_s3_policy.json deleted file mode 100644 index 125e0dc..0000000 --- a/templates/email_alert_notifications/put_and_delete_to_s3_policy.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "s3:getObject", - "s3:putObject", - "s3:deleteObject", - "s3:putObjectAcl" - ], - "Effect": "Allow", - "Resource": "${resource_arn}" - } - ] -} diff --git a/templates/email_alert_notifications/write_to_logs_policy.json b/templates/email_alert_notifications/write_to_logs_policy.json deleted file mode 100644 index 63a7264..0000000 --- a/templates/email_alert_notifications/write_to_logs_policy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource": "arn:aws:logs:*:*:*" - } - ] -}