From dd91246adff57a6a98cb1f305f7d4435db259cca Mon Sep 17 00:00:00 2001 From: mark roberts Date: Tue, 10 Oct 2023 16:19:39 +0100 Subject: [PATCH 01/12] inital code for glue fix --- main.tf | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/main.tf b/main.tf index 076723f0..23cf1e95 100644 --- a/main.tf +++ b/main.tf @@ -279,3 +279,68 @@ resource "aws_lb_target_group" "this" { }, ) } + + +# Glue crawler to update Athena Table +# Role for crawler +resource "aws_iam_role" "lb_glue_crawler" { + name = "ssm-glue-crawler" + assume_role_policy = data.aws_iam_policy_document.lb_glue_crawler_assume.json +} + +data "aws_iam_policy_document" "lb_glue_crawler_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_policy" "lb_glue_crawler" { + name = "LbGlueCrawler" + policy = data.aws_iam_policy_document.lb_glue_crawler.json +} + +data "aws_iam_policy_document" "lb_glue_crawler" { + statement { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject" + ] + resources = [var.existing_bucket_name != "" ? "arn:aws:s3:::${var.existing_bucket_name}/${var.application_name}/AWSLogs/${var.account_number}/*" : "${module.s3-bucket[0].bucket.arn}/${var.application_name}/AWSLogs/${var.account_number}/*"] + } + statement { + effect = "Allow" + actions = [ + "kms:Decrypt" + ] + resources = [aws_kms_key.lb-access-logs.arn] + } +} + +resource "aws_iam_role_policy_attachment" "lb_glue_crawler" { + role = aws_iam_role.lb_glue_crawler.name + policy_arn = aws_iam_policy.lb_glue_crawler.arn +} + +resource "aws_iam_role_policy_attachment" "lb_glue_servicec" { + role = aws_iam_role.lb_glue_crawler.id + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" +} + +# Glue Crawler +resource "aws_glue_crawler" "ssm_resource_sync" { + database_name = aws_athena_database.lb-access-logs.name + name = "lb_resource_sync" + role = aws_iam_role.lb_glue_crawler.arn + schedule = "cron(15 1 ? * MON *)" + + s3_target { + path = "s3://${module.s3-bucket.bucket_name}" + } +} \ No newline at end of file From 2a79d2fc9c516efcf8b02653c82c98d1abb49a64 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 10 Oct 2023 15:24:44 +0000 Subject: [PATCH 02/12] terraform-docs: automated action --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 41a04ad0..0411186e 100644 --- a/README.md +++ b/README.md @@ -171,11 +171,18 @@ If you're looking to raise an issue with this module, please create a new issue | [aws_athena_database.lb-access-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_database) | resource | | [aws_athena_named_query.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource | | [aws_athena_workgroup.lb-access-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_workgroup) | resource | +| [aws_glue_crawler.ssm_resource_sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_crawler) | resource | +| [aws_iam_policy.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.lb_glue_servicec](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_lb.loadbalancer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | | [aws_lb_target_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | | [aws_security_group.lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_elb_service_account.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/elb_service_account) | data source | | [aws_iam_policy_document.bucket_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lb_glue_crawler_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_vpc.shared](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | ## Inputs From 609f7a28879815c904cd5303b24f946494142beb Mon Sep 17 00:00:00 2001 From: mark roberts Date: Tue, 10 Oct 2023 16:30:36 +0100 Subject: [PATCH 03/12] code corrections for tests --- main.tf | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 23cf1e95..109c3882 100644 --- a/main.tf +++ b/main.tf @@ -314,13 +314,6 @@ data "aws_iam_policy_document" "lb_glue_crawler" { ] resources = [var.existing_bucket_name != "" ? "arn:aws:s3:::${var.existing_bucket_name}/${var.application_name}/AWSLogs/${var.account_number}/*" : "${module.s3-bucket[0].bucket.arn}/${var.application_name}/AWSLogs/${var.account_number}/*"] } - statement { - effect = "Allow" - actions = [ - "kms:Decrypt" - ] - resources = [aws_kms_key.lb-access-logs.arn] - } } resource "aws_iam_role_policy_attachment" "lb_glue_crawler" { @@ -335,7 +328,7 @@ resource "aws_iam_role_policy_attachment" "lb_glue_servicec" { # Glue Crawler resource "aws_glue_crawler" "ssm_resource_sync" { - database_name = aws_athena_database.lb-access-logs.name + database_name = aws_athena_database.lb-access-logs[count.index] name = "lb_resource_sync" role = aws_iam_role.lb_glue_crawler.arn schedule = "cron(15 1 ? * MON *)" From b6cc69bb01762256a9228c060a269ddc7408010f Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 09:18:20 +0100 Subject: [PATCH 04/12] changes for module code --- main.tf | 6 +++--- test/unit-test/main.tf | 1 + variables.tf | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 109c3882..1027af12 100644 --- a/main.tf +++ b/main.tf @@ -328,12 +328,12 @@ resource "aws_iam_role_policy_attachment" "lb_glue_servicec" { # Glue Crawler resource "aws_glue_crawler" "ssm_resource_sync" { - database_name = aws_athena_database.lb-access-logs[count.index] + database_name = aws_athena_database.lb-access-logs[0].name name = "lb_resource_sync" role = aws_iam_role.lb_glue_crawler.arn - schedule = "cron(15 1 ? * MON *)" + schedule = var.log_schedule s3_target { - path = "s3://${module.s3-bucket.bucket_name}" + path = "s3://${var.existing_bucket_name}" } } \ No newline at end of file diff --git a/test/unit-test/main.tf b/test/unit-test/main.tf index 389b8397..990c9193 100644 --- a/test/unit-test/main.tf +++ b/test/unit-test/main.tf @@ -76,4 +76,5 @@ module "lb_access_logs_enabled" { idle_timeout = 60 force_destroy_bucket = true lb_target_groups = local.lb_target_groups + log_schedule = "cron(15 1 ? * MON *)" } diff --git a/variables.tf b/variables.tf index 6ed8214b..5ec5266c 100644 --- a/variables.tf +++ b/variables.tf @@ -118,4 +118,9 @@ variable "lb_target_groups" { })), []) })) default = {} +} +variable "log_schedule" { + type = string + default = "cron(15 1 ? * MON *)" + } \ No newline at end of file From e5e48406e895b401dd1d8305ca139b4fcba04b44 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 11 Oct 2023 08:18:52 +0000 Subject: [PATCH 05/12] terraform-docs: automated action --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0411186e..0fbe7b3a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ If you're looking to raise an issue with this module, please create a new issue | [load\_balancer\_type](#input\_load\_balancer\_type) | application or network | `string` | `"application"` | no | | [loadbalancer\_egress\_rules](#input\_loadbalancer\_egress\_rules) | Create new security group with these egress rules for the loadbalancer. Or use the security\_groups var to attach existing group(s) | 
map(object({
    description     = string
    from_port       = number
    to_port         = number
    protocol        = string
    security_groups = list(string)
    cidr_blocks     = list(string)
  }))
| `{}` | no | | [loadbalancer\_ingress\_rules](#input\_loadbalancer\_ingress\_rules) | Create new security group with these ingress rules for the loadbalancer. Or use the security\_groups var to attach existing group(s) | 
map(object({
    description     = string
    from_port       = number
    to_port         = number
    protocol        = string
    security_groups = list(string)
    cidr_blocks     = list(string)
  }))
| `{}` | no | +| [log\_schedule](#input\_log\_schedule) | n/a | `string` | `"cron(15 1 ? * MON *)"` | no | | [public\_subnets](#input\_public\_subnets) | Public subnets | `list(string)` | n/a | yes | | [region](#input\_region) | AWS Region where resources are to be created | `string` | n/a | yes | | [s3\_versioning](#input\_s3\_versioning) | A boolean that determines whether s3 will have versioning | `bool` | `true` | no | From 98058df484e222ea9973333df6e21d1c668a34eb Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 09:27:45 +0100 Subject: [PATCH 06/12] added a comment --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 1027af12..59895b07 100644 --- a/main.tf +++ b/main.tf @@ -316,6 +316,7 @@ data "aws_iam_policy_document" "lb_glue_crawler" { } } +# Glue Crawler Policy resource "aws_iam_role_policy_attachment" "lb_glue_crawler" { role = aws_iam_role.lb_glue_crawler.name policy_arn = aws_iam_policy.lb_glue_crawler.arn From cbaedb9c48accf5e5e8ea9117c16731bcc132ebd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 11 Oct 2023 08:28:55 +0000 Subject: [PATCH 07/12] Commit changes made by code formatters --- variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index 5ec5266c..3e1ed95a 100644 --- a/variables.tf +++ b/variables.tf @@ -120,7 +120,7 @@ variable "lb_target_groups" { default = {} } variable "log_schedule" { - type = string + type = string default = "cron(15 1 ? * MON *)" - + } \ No newline at end of file From a5a009b4e11e3e10e8c10f0c12a14838f75fd6f9 Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 14:08:17 +0100 Subject: [PATCH 08/12] ignore added for the moment on checkov --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 59895b07..71e35ff4 100644 --- a/main.tf +++ b/main.tf @@ -329,6 +329,7 @@ resource "aws_iam_role_policy_attachment" "lb_glue_servicec" { # Glue Crawler resource "aws_glue_crawler" "ssm_resource_sync" { + #checkov:skip=CKV_AWS_195 database_name = aws_athena_database.lb-access-logs[0].name name = "lb_resource_sync" role = aws_iam_role.lb_glue_crawler.arn From 4a0aad18e4b2605972b3e6d5cded2f0b801687f6 Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 14:19:32 +0100 Subject: [PATCH 09/12] checkov correction --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 53ad2cd2..8cc83572 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -4,7 +4,7 @@ on: pull_request: branches: - main - +permissions: read-all jobs: docs: runs-on: ubuntu-latest From ec8d532bb9df208c9053ab89958edff00fa06e46 Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 14:28:17 +0100 Subject: [PATCH 10/12] added skip to the workflow --- .github/workflows/documentation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 8cc83572..f44511b6 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,10 +1,11 @@ +#checkov:skip:CKV2_GHA_1:Comment name: Generate Terraform README docs on: workflow_dispatch: pull_request: branches: - main -permissions: read-all + jobs: docs: runs-on: ubuntu-latest From e6be90fd1d4af6bb28eedaf3c2029b8237a00144 Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 14:36:23 +0100 Subject: [PATCH 11/12] trying to fix checkoh error --- .github/workflows/documentation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f44511b6..279379a6 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,10 +1,11 @@ -#checkov:skip:CKV2_GHA_1:Comment name: Generate Terraform README docs on: workflow_dispatch: pull_request: branches: - main + +permissions: read-all jobs: docs: From f85499b25e2dbcf3f7237f358104d18ca21628fa Mon Sep 17 00:00:00 2001 From: mark roberts Date: Wed, 11 Oct 2023 15:34:09 +0100 Subject: [PATCH 12/12] spelling correction --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 71e35ff4..66c6efa6 100644 --- a/main.tf +++ b/main.tf @@ -322,7 +322,7 @@ resource "aws_iam_role_policy_attachment" "lb_glue_crawler" { policy_arn = aws_iam_policy.lb_glue_crawler.arn } -resource "aws_iam_role_policy_attachment" "lb_glue_servicec" { +resource "aws_iam_role_policy_attachment" "lb_glue_service" { role = aws_iam_role.lb_glue_crawler.id policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" }