Skip to content
This repository has been archived by the owner on Dec 17, 2021. It is now read-only.

Added Log4J for internal CT instance #8

Merged
merged 5 commits into from
Mar 31, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [1.3.0] - 2020-03-31
### Added
- Log4j XML for internal Circus Train instance.

## [1.2.1] - 2020-01-16
### Added
- A new argument `orphaned_data_strategy` to use for handling stale data during replication.
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
| aws\_region | AWS region to use for resources. | string | n/a | yes |
| cpu | The number of CPU units to reserve for the Shunting Yard container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"1024"` | no |
| ct\_common\_config\_yaml | Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train. | string | n/a | yes |
| ct\_log4j\_xml | Log4j XML file to be passed to internal Circus Train instance to configure logging. | string | n/a | yes |
| docker\_image | Full path of Shunting Yard Docker image. | string | n/a | yes |
| docker\_registry\_auth\_secret\_name | Docker Registry authentication SecretManager secret name. | string | `""` | no |
| docker\_version | Shunting Yard Docker image version. | string | n/a | yes |
Expand Down
4 changes: 2 additions & 2 deletions cloudwatch.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down Expand Up @@ -98,7 +98,7 @@ data "template_file" "sqs_widgets" {
"region": "${var.aws_region}",
"title": "Shunting Yard SQS Age of Oldest Message (s)"
}
}
}
EOF
}

Expand Down
2 changes: 1 addition & 1 deletion common.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion ecs.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion iam-policy-s3-buckets.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion iam-policy-secretsmanager.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion iam-policy-sqs.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion iam.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion sg.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion sns.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
2 changes: 1 addition & 1 deletion sqs.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down
3 changes: 2 additions & 1 deletion templates.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down Expand Up @@ -28,6 +28,7 @@ data "template_file" "shuntingyard" {
loggroup = "${aws_cloudwatch_log_group.shuntingyard_ecs.name}"
shuntingyard_config_yaml = "${base64encode(data.template_file.shuntingyard_config_yaml.rendered)}"
ct_common_config_yaml = "${base64encode(var.ct_common_config_yaml)}"
ct_log4j_xml = "${base64encode(var.ct_log4j_xml)}"

#to instruct ECS to use repositoryCredentials for private docker registry
docker_auth = "${var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("", data.aws_secretsmanager_secret.docker_registry.*.arn))}"
Expand Down
4 changes: 2 additions & 2 deletions templates/shuntingyard-config.yml.tmpl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
# Copyright (C) 2019 Expedia, Inc.
# Copyright (C) 2020 Expedia, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");

source-catalog:
Expand All @@ -14,6 +14,6 @@ event-receiver:
sqs.queue: ${shuntingyard_sqs_queue}
sqs.wait.time.seconds: ${shuntingyard_sqs_queue_wait_timeout}
source-table-filter:
table-names:
table-names:
${selected_tables}
${orphaned_data_strategy}
4 changes: 4 additions & 0 deletions templates/shuntingyard.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
{
"name": "CT_COMMON_CONFIG_YAML",
"value": "${ct_common_config_yaml}"
},
{
"name": "CT_LOG4J_XML",
"value": "${ct_log4j_xml}"
}
]
}
Expand Down
9 changes: 7 additions & 2 deletions variables.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down Expand Up @@ -67,6 +67,11 @@ variable "ct_common_config_yaml" {
type = "string"
}

variable "ct_log4j_xml" {
description = "Log4j XML file to be passed to internal Circus Train instance to configure logging."
type = "string"
}

variable "allowed_s3_buckets" {
description = "List of S3 Buckets to which Shunting Yard will have read-write access."
type = "list"
Expand Down Expand Up @@ -101,7 +106,7 @@ variable "shuntingyard_sqs_queue_stale_messages_timeout" {

variable "selected_tables" {
description = <<EOF
Tables selected for Shunting Yard Replication.
Tables selected for Shunting Yard Replication.
Supported Format: [ "database_1.table_1", "database_2.table_2" ]
Wildcards are not supported, i.e. you need to specify each table explicitly.
EOF
Expand Down
2 changes: 1 addition & 1 deletion version.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (C) 2019 Expedia, Inc.
* Copyright (C) 2020 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/
Expand Down