From 659821dc19c3966d1ffbee21b857933ccc092f15 Mon Sep 17 00:00:00 2001 From: Ryan Tan <1265924+ryantanjunming@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:43:02 +0800 Subject: [PATCH] Feature Update: Firehose, Data Reference Resources (s3, iam), Ingress Url & AP3 region changes (#175) * main changes added * wrong variable * AP3 readme * update variables description * domain naming and fixes * changelog and readme * more readme * example files * fix issue with tests * change private_key to api_key in firehose-logs test * fix readme and descriptions * update changelog --------- Co-authored-by: guyrenny --- CHANGELOG.md | 10 +++ examples/firehose-logs/main.tf | 2 +- examples/firehose-logs/variables.tf | 4 +- modules/firehose-logs/README.md | 30 +++---- modules/firehose-logs/main.tf | 98 +++++++++++++--------- modules/firehose-logs/variables.tf | 42 +++++++--- modules/firehose-metrics/README.md | 30 +++---- modules/firehose-metrics/main.tf | 5 +- modules/firehose-metrics/variables.tf | 8 +- tests/firehose-logs/firehose-logs.tf | 4 +- tests/firehose-metrics/firehose-metrics.tf | 2 +- 11 files changed, 144 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77da4099..ba965313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## v1.0.107 +#### **firehose-logs & firehose-metrics** +### 💡 Enhancements +- Add AP3 region to the list of regions +- Added custom naming for global resources +- Added ability to import global resources (s3 & iam) +### 🛑 Breaking changes 🛑 +- For firehose-logs & firehose-metrics, Update variables: `coralogix_region` values regions from [Europe, Europe2, India, Singapore, US, US2] to [EU1, EU2, AP1, AP2, AP3, US1, US2] +- Update variables: `private_key` renamed to `api_key` with type `string` instead of `any`. + ## v1.0.106 #### **msk-data-stream** ### 💡 Enhancements diff --git a/examples/firehose-logs/main.tf b/examples/firehose-logs/main.tf index 55431727..1418f2e8 100644 --- a/examples/firehose-logs/main.tf +++ b/examples/firehose-logs/main.tf @@ -1,7 +1,7 @@ module "cloudwatch_firehose_logs_coralogix" { source = "coralogix/aws/coralogix//modules/firehose-logs" firehose_stream = var.firehose_stream - private_key = var.private_key + api_key = var.api_key coralogix_region = var.coralogix_region integration_type_logs = "Default" source_type_logs = "DirectPut" diff --git a/examples/firehose-logs/variables.tf b/examples/firehose-logs/variables.tf index f0dbce0b..e31b9c79 100644 --- a/examples/firehose-logs/variables.tf +++ b/examples/firehose-logs/variables.tf @@ -9,9 +9,9 @@ variable "coralogix_region" { description = "The region of the Coralogix account" } -variable "private_key" { +variable "api_key" { type = string - description = "Coralogix account logs private key" + description = "Coralogix account api key" sensitive = true } diff --git a/modules/firehose-logs/README.md b/modules/firehose-logs/README.md index 97a561ef..359f342d 100644 --- a/modules/firehose-logs/README.md +++ b/modules/firehose-logs/README.md @@ -46,23 +46,25 @@ The application name and subsystem name by default is the firehose delivery stre # Coralogix account region The coralogix region variable accepts one of the following regions: -* Europe -* Europe2 -* India -* Singapore -* US +* EU1 +* EU2 +* AP1 +* AP2 +* AP3 +* US1 * US2 ### Coralogix Regions & Description. -| Region | Domain | Endpoint | -|-----------|------------------------|---------------------------------------------------------| -| Europe | `coralogix.com` | `https://firehose-ingress.coralogix.com/firehose` | -| Europe2 | `eu2.coralogix.com` | `https://firehose-ingress.eu2.coralogix.com/firehose` | -| India | `coralogix.in` | `https://firehose-ingress.app.coralogix.in/firehose` | -| Singapore | `coralogixsg.com` | `https://firehose-ingress.coralogixsg.com/firehose` | -| US | `coralogix.us` | `https://firehose-ingress.coralogix.us/firehose` | -| US2 | `cx498.coralogix.com` | `https://firehose-ingress.cx498.coralogix.com/firehose` | +| Region | Domain | Endpoint | +|-----------|------------------------|----------------------------------------------------| +| EU1 | `coralogix.com` | `https://ingress.coralogix.com/aws/firehose` | +| EU2 | `eu2.coralogix.com` | `https://ingress.eu2.coralogix.com/aws/firehose` | +| AP1 | `coralogix.in` | `https://ingress.app.coralogix.in/aws/firehose` | +| AP2 | `coralogixsg.com` | `https://ingress.coralogixsg.com/aws/firehose` | +| AP3 | `ap3.coralogix.com` | `https://ingress.ap3.coralogix.com/aws/firehose` | +| US1 | `coralogix.us` | `https://ingress.coralogix.us/aws/firehose` | +| US2 | `cx498.coralogix.com` | `https://ingress.cx498.coralogix.com/aws/firehose` | ### Custom Domain It is possible to pass a custom coralogix domain by using the `custom_domain` variable. @@ -84,7 +86,7 @@ It is possible to pass a custom coralogix domain by using the `custom_domain` va | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [coralogix\_region](#input\_coralogix\_region) | Coralogix account region: Europe, Europe2, India, Singapore, US, US2 [exact] | `any` | n/a | yes | +| [coralogix\_region](#input\_coralogix\_region) | Coralogix account region: EU1, EU2, AP1, AP2, AP3, US1, US2 [exact] | `any` | n/a | yes | | [private_key](#input\_private_key) | Coralogix account logs private key | `any` | n/a | yes | | [firehose\_stream](#input\_firehose\_stream) | AWS Kinesis firehose delivery stream name | `string` | n/a | yes | | [application_name](#input\_application_name) | The name of your application in Coralogix | `string` | n/a | yes | diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf index 45c9e068..7631fd0f 100644 --- a/modules/firehose-logs/main.tf +++ b/modules/firehose-logs/main.tf @@ -25,8 +25,13 @@ locals { custom_endpoint = local.endpoint_url }) : var.user_supplied_tags - # default namings - s3_logs_backup_bucket_name = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-logs" + # global resource referecing + s3_backup_bucket_arn = var.existing_s3_backup != null ? one(data.aws_s3_bucket.exisiting_s3_bucket[*].arn) : one(aws_s3_bucket.new_s3_bucket[*].arn) + firehose_iam_role_arn = var.existing_firehose_iam != null ? one(data.aws_iam_role.existing_firehose_iam[*].arn) : one(aws_iam_role.new_firehose_iam[*].arn) + + #new global resource namings + new_s3_backup_bucket_name = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-logs" + new_firehose_iam_name = var.firehose_iam_custom_name != null ? var.firehose_iam_custom_name : "${var.firehose_stream}-firehose-logs-iam" } data "aws_caller_identity" "current_identity" {} @@ -57,13 +62,20 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" { log_group_name = aws_cloudwatch_log_group.firehose_loggroup.name } -resource "aws_s3_bucket" "firehose_bucket" { - tags = merge(local.tags, { Name = local.s3_logs_backup_bucket_name }) - bucket = local.s3_logs_backup_bucket_name +data "aws_s3_bucket" "exisiting_s3_bucket" { + count = var.existing_s3_backup != null ? 1 : 0 + bucket = var.existing_s3_backup +} + +resource "aws_s3_bucket" "new_s3_bucket" { + count = var.existing_s3_backup != null ? 0 : 1 + tags = merge(local.tags, { Name = local.new_s3_backup_bucket_name }) + bucket = local.new_s3_backup_bucket_name } resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" { - bucket = aws_s3_bucket.firehose_bucket.id + count = var.existing_s3_backup != null ? 0 : 1 + bucket = one(aws_s3_bucket.new_s3_bucket[*].id) block_public_acls = true block_public_policy = true @@ -75,9 +87,15 @@ resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" { # Firehose Logs Stream ################################################################################ -resource "aws_iam_role" "firehose_to_coralogix" { - tags = local.tags - name = "${var.firehose_stream}-firehose-logs" +data "aws_iam_role" "existing_firehose_iam" { + count = var.existing_firehose_iam != null ? 1 : 0 + name = var.existing_firehose_iam +} + +resource "aws_iam_role" "new_firehose_iam" { + count = var.existing_firehose_iam != null ? 0 : 1 + tags = local.tags + name = local.new_firehose_iam_name assume_role_policy = jsonencode({ "Version" = "2012-10-17", "Statement" = [ @@ -91,7 +109,7 @@ resource "aws_iam_role" "firehose_to_coralogix" { ] }) inline_policy { - name = "${var.firehose_stream}-firehose" + name = local.new_firehose_iam_name policy = jsonencode({ "Version" = "2012-10-17", "Statement" = [ @@ -106,8 +124,8 @@ resource "aws_iam_role" "firehose_to_coralogix" { "s3:PutObject" ], "Resource" = [ - aws_s3_bucket.firehose_bucket.arn, - "${aws_s3_bucket.firehose_bucket.arn}/*" + "${local.s3_backup_bucket_arn}", + "${local.s3_backup_bucket_arn}/*" ] }, { @@ -121,12 +139,12 @@ resource "aws_iam_role" "firehose_to_coralogix" { "Resource" = "arn:aws:kinesis:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:stream/*" }, { - "Effect" = "Allow", - "Action" = [ - "*" + "Effect" : "Allow", + "Action" : [ + "logs:PutLogEvents" ], - "Resource" = [ - aws_cloudwatch_log_group.firehose_loggroup.arn + "Resource" : [ + "${aws_cloudwatch_log_group.firehose_loggroup.arn}" ] } ] @@ -134,32 +152,51 @@ resource "aws_iam_role" "firehose_to_coralogix" { } } +# Add additional policies to the firehose IAM role +resource "aws_iam_role_policy_attachment" "policy_attachment_firehose" { + count = var.existing_firehose_iam != null ? 0 : 1 + role = one(aws_iam_role.new_firehose_iam[*].name) + policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisFirehoseFullAccess" +} + +resource "aws_iam_role_policy_attachment" "policy_attachment_kinesis" { + count = var.existing_firehose_iam != null ? 0 : 1 + role = one(aws_iam_role.new_firehose_iam[*].name) + policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisReadOnlyAccess" +} + +resource "aws_iam_role_policy_attachment" "policy_attachment_cloudwatch" { + count = var.existing_firehose_iam != null ? 0 : 1 + role = one(aws_iam_role.new_firehose_iam[*].name) + policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" +} + resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" { tags = local.tags - name = "${var.firehose_stream}-logs" + name = var.firehose_stream destination = "http_endpoint" dynamic "kinesis_source_configuration" { for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [1] : [] content { kinesis_stream_arn = var.kinesis_stream_arn - role_arn = aws_iam_role.firehose_to_coralogix.arn + role_arn = local.firehose_iam_role_arn } } http_endpoint_configuration { url = local.endpoint_url name = "Coralogix" - access_key = var.private_key + access_key = var.api_key buffering_size = 6 buffering_interval = 60 s3_backup_mode = "FailedDataOnly" - role_arn = aws_iam_role.firehose_to_coralogix.arn + role_arn = local.firehose_iam_role_arn retry_duration = 300 s3_configuration { - role_arn = aws_iam_role.firehose_to_coralogix.arn - bucket_arn = aws_s3_bucket.firehose_bucket.arn + role_arn = local.firehose_iam_role_arn + bucket_arn = local.s3_backup_bucket_arn buffering_size = 5 buffering_interval = 300 compression_format = "GZIP" @@ -200,18 +237,3 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" { } } } - -resource "aws_iam_role_policy_attachment" "example_policy_attachment" { - role = aws_iam_role.firehose_to_coralogix.name - policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisFirehoseFullAccess" -} - -resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" { - role = aws_iam_role.firehose_to_coralogix.name - policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisReadOnlyAccess" -} - -resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" { - role = aws_iam_role.firehose_to_coralogix.name - policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" -} diff --git a/modules/firehose-logs/variables.tf b/modules/firehose-logs/variables.tf index aa2d0ad0..9aabb202 100644 --- a/modules/firehose-logs/variables.tf +++ b/modules/firehose-logs/variables.tf @@ -1,14 +1,14 @@ variable "coralogix_region" { - description = "Coralogix account region: Europe, Europe2, India, Singapore, US, US2" + description = "Coralogix account region: EU1, EU2, AP1, AP2, AP3, US1, US2" type = string validation { - condition = contains(["Europe", "Europe2", "India", "Singapore", "US", "US2", "AP3"], var.coralogix_region) - error_message = "The coralogix region must be one of these values: [Europe, Europe2, India, Singapore, US, US2, AP3]." + condition = contains(["EU1", "EU2", "AP1", "AP2", "AP3", "US1", "US2"], var.coralogix_region) + error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, AP3, US1, US2]." } } -variable "private_key" { - description = "Coralogix account private key" +variable "api_key" { + description = "Coralogix account api key" type = string sensitive = true } @@ -37,7 +37,7 @@ variable "cloudwatch_retention_days" { } variable "custom_domain" { - description = "Custom domain for Coralogix firehose integration endpoint (e.g. private.coralogix.net:8443 for https://firehose-ingress.private.coralogix.net:8443/firehose)" + description = "Custom domain for Coralogix firehose integration endpoints, does not work for privatelink (e.g. cust.coralogix-123.net:8443 for https://ingress.cust.coralogix-123.net:8443/aws/firehose)" type = string default = null } @@ -60,6 +60,30 @@ variable "integration_type_logs" { default = null } +variable "s3_backup_custom_name" { + description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-logs' will be used" + type = string + default = null +} + +variable "existing_s3_backup" { + description = "Use an existing S3 bucket to use as a backup bucket" + type = string + default = null +} + +variable "firehose_iam_custom_name" { + description = "Set the name of the firehose IAM role & policy, otherwise variable '{firehose_stream}-firehose-logs-iam' will be used" + type = string + default = null +} + +variable "existing_firehose_iam" { + description = "Use an existing IAM role to use as a firehose role" + type = string + default = null +} + variable "user_supplied_tags" { description = "Tags supplied by the user to populate to all generated resources" type = map(string) @@ -71,9 +95,3 @@ variable "override_default_tags" { type = bool default = false } - -variable "s3_backup_custom_name" { - description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-logs' will be used" - type = string - default = null -} diff --git a/modules/firehose-metrics/README.md b/modules/firehose-metrics/README.md index 68bb22a2..7ccdc643 100644 --- a/modules/firehose-metrics/README.md +++ b/modules/firehose-metrics/README.md @@ -146,23 +146,25 @@ The application name and subsystem name by default is the firehose delivery stre # Coralogix account region The coralogix region variable accepts one of the following regions: -* Europe -* Europe2 -* India -* Singapore -* US +* EU1 +* EU2 +* AP1 +* AP2 +* AP3 +* US1 * US2 ### Coralogix Regions & Description. -| Region | Domain | Endpoint | -|-----------|------------------------|---------------------------------------------------------| -| Europe | `coralogix.com` | `https://firehose-ingress.coralogix.com/firehose` | -| Europe2 | `eu2.coralogix.com` | `https://firehose-ingress.eu2.coralogix.com/firehose` | -| India | `coralogix.in` | `https://firehose-ingress.app.coralogix.in/firehose` | -| Singapore | `coralogixsg.com` | `https://firehose-ingress.coralogixsg.com/firehose` | -| US | `coralogix.us` | `https://firehose-ingress.coralogix.us/firehose` | -| US2 | `cx498.coralogix.com` | `https://firehose-ingress.cx498.coralogix.com/firehose` | +| Region | Domain | Endpoint | +|-----------|------------------------|----------------------------------------------------| +| EU1 | `coralogix.com` | `https://ingress.coralogix.com/aws/firehose` | +| EU2 | `eu2.coralogix.com` | `https://ingress.eu2.coralogix.com/aws/firehose` | +| AP1 | `coralogix.in` | `https://ingress.app.coralogix.in/aws/firehose` | +| AP2 | `coralogixsg.com` | `https://ingress.coralogixsg.com/aws/firehose` | +| AP3 | `ap3.coralogix.com` | `https://ingress.ap3.coralogix.com/aws/firehose` | +| US1 | `coralogix.us` | `https://ingress.coralogix.us/aws/firehose` | +| US2 | `cx498.coralogix.com` | `https://ingress.cx498.coralogix.com/aws/firehose` | ### Custom endpoints It is possible to pass a custom firehose ingress endpoint with by using the `coralogix_firehose_custom_endpoint` variable. @@ -191,7 +193,7 @@ then the CloudWatch metric stream must be configured with the same format, confi | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [coralogix\_region](variables.tf#L1) | Coralogix account region: Europe, Europe2, India, Singapore, US, US2 [exact] | `any` | n/a | yes | +| [coralogix\_region](variables.tf#L1) | Coralogix account region: EU1, EU2, AP1, AP2, AP3, US1, US2 [exact] | `any` | n/a | yes | | [api\_key](variables.tf#L10) | Coralogix account logs api key | `string` | n/a | yes | | [firehose\_stream](variables.tf#L16) | AWS Kinesis firehose delivery stream name | `string` | n/a | yes | | [application\_name](variables.tf#L21) | The name of your application in Coralogix | `string` | n/a | yes | diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf index ffedb1b2..934ea6dd 100644 --- a/modules/firehose-metrics/main.tf +++ b/modules/firehose-metrics/main.tf @@ -33,7 +33,6 @@ locals { # default resource namings lambda_processor_name = var.lambda_processor_custom_name != null ? var.lambda_processor_custom_name : "${var.firehose_stream}-metrics-transform" - firehose_stream_name = var.firehose_stream cloud_watch_metric_stream_name = var.cloudwatch_metric_stream_custom_name != null ? var.cloudwatch_metric_stream_custom_name : "${var.firehose_stream}-cw" #new global resource namings @@ -58,7 +57,7 @@ resource "random_string" "this" { resource "aws_cloudwatch_log_group" "firehose_loggroup" { tags = local.tags - name = "/aws/kinesisfirehosemetrics/${local.firehose_stream_name}" + name = "/aws/kinesisfirehosemetrics/${var.firehose_stream}" retention_in_days = var.cloudwatch_retention_days } @@ -301,7 +300,7 @@ resource "aws_lambda_function" "lambda_processor" { resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_metrics" { tags = local.tags - name = local.firehose_stream_name + name = var.firehose_stream destination = "http_endpoint" http_endpoint_configuration { diff --git a/modules/firehose-metrics/variables.tf b/modules/firehose-metrics/variables.tf index 5fce073d..1b91389f 100644 --- a/modules/firehose-metrics/variables.tf +++ b/modules/firehose-metrics/variables.tf @@ -1,9 +1,9 @@ variable "coralogix_region" { - description = "Coralogix account region: Europe, Europe2, India, Singapore, US, US2" + description = "Coralogix account region: EU1, EU2, AP1, AP2, AP3, US1, US2" type = string validation { - condition = contains(["Europe", "Europe2", "India", "Singapore", "US", "US2", "AP3"], var.coralogix_region) - error_message = "The coralogix region must be one of these values: [Europe, Europe2, India, Singapore, US, US2, AP3]." + condition = contains(["EU1", "EU2", "AP1", "AP2", "AP3", "US1", "US2"], var.coralogix_region) + error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, AP3, US1, US2]." } } @@ -37,7 +37,7 @@ variable "cloudwatch_retention_days" { } variable "custom_domain" { - description = "Custom domain for Coralogix firehose integration endpoint (e.g. private.coralogix.net:8443 for https://firehose-ingress.private.coralogix.net:8443/firehose)" + description = "Custom domain for Coralogix firehose integration endpoints, does not work for privatelink (e.g. cust.coralogix-123.net:8443 for https://ingress.cust.coralogix-123.net:8443/aws/firehose)" type = string default = null } diff --git a/tests/firehose-logs/firehose-logs.tf b/tests/firehose-logs/firehose-logs.tf index 3a88748a..445ddc21 100644 --- a/tests/firehose-logs/firehose-logs.tf +++ b/tests/firehose-logs/firehose-logs.tf @@ -15,7 +15,7 @@ provider "aws" { module "firehose-logs" { source = "../../modules/firehose-logs" - coralogix_region = "Europe" - private_key = "{{ secrets.TESTING_PRIVATE_KEY }}" + coralogix_region = "EU1" + api_key = "{{ secrets.TESTING_PRIVATE_KEY }}" firehose_stream = "test-stream" } diff --git a/tests/firehose-metrics/firehose-metrics.tf b/tests/firehose-metrics/firehose-metrics.tf index b185d46d..32bacaa1 100644 --- a/tests/firehose-metrics/firehose-metrics.tf +++ b/tests/firehose-metrics/firehose-metrics.tf @@ -15,7 +15,7 @@ provider "aws" { module "firehose-metrics" { source = "../../modules/firehose-metrics" - coralogix_region = "Europe" + coralogix_region = "EU1" api_key = "{{ secrets.TESTING_PRIVATE_KEY }}" firehose_stream = "test-stream" }