Skip to content

Commit

Permalink
Complete tf surgery; Identify all TODOs in golang
Browse files Browse the repository at this point in the history
For #25
  • Loading branch information
rivernews committed Sep 25, 2022
1 parent ec37c70 commit d816fe6
Show file tree
Hide file tree
Showing 19 changed files with 348 additions and 216 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
**/builds/**

lambda_golang/landing
lambda_golang/landing_s3_trigger
lambda_golang/landing_metadata_cronjob
lambda_golang/stories
lambda_golang/landing_metadata
lambda_golang/story
lambda_golang/stories_finalizer
venv

# Binaries for programs and plugins
Expand Down
6 changes: 4 additions & 2 deletions cloud_environments/terraform.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ set +o allexport
if (
cd $GOLANG_SRC_DIR && \
go build ./cmd/landing && \
go build ./cmd/landing_metadata && \
go build ./cmd/landing_s3_trigger && \
go build ./cmd/landing_metadata_cronjob && \
go build ./cmd/stories && \
go build ./cmd/story && \
go build ./cmd/stories_finalizer && \
cd $PYTHON_SRC_DIR && python -m compileall layer src
); then
cd $DEPLOY_DIR
Expand All @@ -37,7 +39,7 @@ if (
# https://github.com/terraform-aws-modules/terraform-aws-step-functions/issues/20
# terraform "$@" \
# -target=module.main.module.scraper_lambda \
# -target=module.main.module.landing_parse_metadata_lambda
# -target=module.main.module.landing_metadata_cronjob_lambda

terraform "$@"
else
Expand Down
2 changes: 1 addition & 1 deletion cloud_module/dynamodb/table.tf
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
resource "aws_ssm_parameter" "media_table" {
name = "/app/media-literacy/table"
type = "String"
value = aws_dynamodb_table.media_table.arn
value = "${aws_dynamodb_table.media_table.arn},${aws_dynamodb_table.media_table.id}"
}

// https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table#attributes-reference
Expand Down
4 changes: 4 additions & 0 deletions cloud_module/pipeline/global_ssm.tf
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,8 @@ data aws_ssm_parameter media_table {
locals {
newssite_economy_tokens = split(",", data.aws_ssm_parameter.newssite_economy.value)
newssite_economy_alias = local.newssite_economy_tokens[2]

_media_table_tokens = split(",", data.aws_ssm_parameter.media_table)
media_table_arn = local._media_table_tokens[0]
media_table_id = local._media_table_tokens[1]
}
38 changes: 0 additions & 38 deletions cloud_module/pipeline/landing_s3_trigger.tf

This file was deleted.

152 changes: 152 additions & 0 deletions cloud_module/pipeline/s3_triggers.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
resource "aws_s3_bucket_notification" "bucket_notification" {
bucket = data.aws_s3_bucket.archive.id

lambda_function {
lambda_function_arn = module.landing_s3_trigger_lambda.lambda_function_arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "${local.newssite_economy_alias}/"
filter_suffix = "landing.html"
}

lambda_function {
lambda_function_arn = module.landing_metadata_s3_trigger_lambda.lambda_function_arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "${local.newssite_economy_alias}/"
filter_suffix = "/metadata.json"
}

depends_on = [
aws_lambda_permission.allow_bucket_trigger_by_landing,
aws_lambda_permission.allow_bucket_trigger_by_landing_metadata
]
}

resource "aws_lambda_permission" "allow_bucket_trigger_by_landing" {
statement_id = "AllowExecutionFromS3Bucket"
action = "lambda:InvokeFunction"
function_name = module.landing_s3_trigger_lambda.lambda_function_arn
principal = "s3.amazonaws.com"
source_arn = data.aws_s3_bucket.archive.arn
}

resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" {
statement_id = "AllowExecutionFromS3Bucket"
action = "lambda:InvokeFunction"
function_name = module.landing_metadata_s3_trigger_lambda.lambda_function_arn
principal = "s3.amazonaws.com"
source_arn = data.aws_s3_bucket.archive.arn
}

module "landing_s3_trigger_lambda" {
source = "terraform-aws-modules/lambda/aws"
create_function = true
function_name = "${local.project_name}-landing-s3-trigger-lambda"
description = "Put a landing page in db"
handler = "landing_s3_trigger"
runtime = "go1.x"

source_path = [{
path = "${var.repo_dir}/lambda_golang/"
commands = ["${local.go_build_flags} go build ./cmd/landing_s3_trigger", ":zip"]
patterns = ["landing_s3_trigger"]
}]

timeout = 900
cloudwatch_logs_retention_in_days = 7
publish = true

attach_policy_statements = true
policy_statements = {
allow_db_put = {
effect = "Allow",
actions = [
"dynamodb:PutItem",
],
resources = [media_table_arn]
}
}

environment_variables = {
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOG_LEVEL = "DEBUG"
DEBUG = "true"
DYNAMODB_TABLE_ID = media_table_id
}

tags = {
Project = local.project_name
}
}

module "landing_metadata_s3_trigger_lambda" {
source = "terraform-aws-modules/lambda/aws"

create_function = true
function_name = "${local.project_name}-fetch-stories"
description = "Fetch ${local.project_name} stories; triggered by metadata.json creation"
handler = "stories"
runtime = "go1.x"
source_path = [{
path = "${var.repo_dir}/lambda_golang/"
commands = ["${local.go_build_flags} go build ./cmd/stories", ":zip"]
patterns = ["stories"]
}]
publish = true

timeout = 900
cloudwatch_logs_retention_in_days = 7

reserved_concurrent_executions = -1

# allow lambda to invoke step function
attach_policy_json = true
policy_json = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"states:StartExecution"
],
"Resource": ["${module.batch_stories_sfn.state_machine_arn}"]
}
]
}
EOF

attach_policy_statements = true
policy_statements = {
s3_archive_bucket = {
effect = "Allow",
actions = [
"s3:GetObject"
],
resources = [
"${data.aws_s3_bucket.archive.arn}/*",
]
}
s3_archive_bucket_check_404 = {
effect = "Allow",
actions = [
"s3:ListBucket",
],
resources = [
"${data.aws_s3_bucket.archive.arn}",
]
}
}

environment_variables = {
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOGLEVEL = "DEBUG"
ENV = local.environment

S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
SFN_ARN = module.batch_stories_sfn.state_machine_arn
}

tags = {
Project = local.project_name
}
}
83 changes: 83 additions & 0 deletions cloud_module/pipeline/scheduler.tf
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,86 @@ data "aws_iam_policy_document" "scheduler" {
}
}
}


resource "aws_cloudwatch_event_rule" "landing_metadata_scheduler" {
count = var.environment_name == "" ? 1 : 0

name = "${local.project_name}-schedule-start-metadata-for-landing"
# schedule experssion
# https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html
schedule_expression = "rate(1 hours)"
description = "Every hour to give courtesy to the website"
}

resource "aws_cloudwatch_event_target" "landing_metadata_scheduler_event_target" {
count = var.environment_name == "" ? 1 : 0

target_id = "${local.project_name}-schedule-start-metadata-for-landing-event-target"
rule = aws_cloudwatch_event_rule.landing_metadata_scheduler.0.name
arn = module.landing_metadata_cronjob_lambda.lambda_function_arn
}

module landing_metadata_cronjob_lambda {
source = "terraform-aws-modules/lambda/aws"
create_function = true
function_name = "${local.project_name}-batch-stories-fetch-parse"
description = "Query landing pages in db; compute & archive their metadata"
handler = "landing_metadata_cronjob"
runtime = "go1.x"

source_path = [{
path = "${var.repo_dir}/lambda_golang/"
commands = ["${local.go_build_flags} go build ./cmd/landing_metadata_cronjob", ":zip"]
patterns = ["landing_metadata_cronjob"]
}]

timeout = 900
cloudwatch_logs_retention_in_days = 7

publish = true

attach_policy_statements = true
policy_statements = {
allow_db_query = {
effect = "Allow",
actions = [
"dynamodb:Query",
"dynamodb:UpdateItem",
],
resources = [media_table_arn]
}
s3_archive_bucket = {
effect = "Allow",
actions = [
"s3:PutObject",
],
resources = [
"${data.aws_s3_bucket.archive.arn}/*",
]
}
# enable getting 404 instead of 403 in case of not found
# https://stackoverflow.com/a/19808954/9814131
s3_archive_bucket_check_404 = {
effect = "Allow",
actions = [
"s3:ListBucket",
],
resources = [
"${data.aws_s3_bucket.archive.arn}",
]
}
}

environment_variables = {
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOG_LEVEL = "DEBUG"
DEBUG = "true"
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
DYNAMODB_TABLE_ID = media_table_id
}

tags = {
Project = local.project_name
}
}
6 changes: 6 additions & 0 deletions cloud_module/pipeline/sfn_def/batch_stories_def.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@
}
}
},
"Next": "Stories-Finalizer",
"End": false
},
"Stories-Finalizer": {
"Type":"Task",
"Resource": "${STORIES_FINALIZER_LAMBDA_ARN}",
"End": true
}
}
Expand Down
Loading

0 comments on commit d816fe6

Please sign in to comment.