Merge pull request #3 from HotelsDotCom/graphite-changes

Adding ct-config option for providing Common CT configurations to internal CT Instance

CHANGELOG.md

@@ -3,7 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.0.0] - TBD
+## [1.1.0] - TBD
+### Added
+- New variable for passing Circus Train common configurations to Shunting Yard. eg. Graphite.
 
+## [1.0.0] - 2019-02-06
 ### Added
 - First version of Shunting Yard Integration with Apiary.

README.md

@@ -8,20 +8,21 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 ## Variables
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| aws_region | AWS region to use for resources. | string | - | yes |
-| cpu | The number of CPU units to reserve for the Shunting Yard container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `1024` | no |
-| memory | The amount of memory (in MiB) used to allocate for the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `4096` | no |
-| docker_image | Full path to Shunting Yard Docker image. | string | - | yes |
-| docker_version | Shunting Yard Docker image version. | string | - | yes |
-| allowed_s3_buckets | List of S3 Buckets to which Shunting Yard will have read-write access. eg. `["bucket-1", "bucket-2"]` | list | `<list>` | no |
-| instance_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `` | no |
-| subnets | ECS container subnets. | list | - | yes |
+| allowed\_s3\_buckets | List of S3 Buckets to which Shunting Yard will have read-write access. eg. `["bucket-1", "bucket-2"]`. | list | `n/a` | yes |
+| aws\_region | AWS region to use for resources. | string | n/a | yes |
+| cpu | The number of CPU units to reserve for the Shunting Yard container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"1024"` | no |
+| ct\_common\_config\_yaml | Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train. | string | n/a | yes |
+| docker\_image | Full path of Shunting Yard Docker image. | string | n/a | yes |
+| docker\_version | Shunting Yard Docker image version. | string | n/a | yes |
+| instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no |
+| memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no |
+| metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes |
+| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | n/a | yes |
+| source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes |
+| subnets | ECS container subnets. | list | n/a | yes |
 | tags | A map of tags to apply to resources. | map | `<map>` | no |
-| vpc_id | VPC ID. | string | - | yes |
-| source_metastore_uri | Source Metastore URI for Shunting Yard. | string | - | yes |
-| target_metastore_uri | Target Metastore URI for Shunting Yard. | string | - | yes |
-| metastore_events_sns_topic | SNS Topic for Hive Metastore events. | string | - | yes |
-| selected_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | - | yes |
+| target\_metastore\_uri | Target Metastore URI for Shunting Yard. | string | n/a | yes |
+| vpc\_id | VPC ID. | string | n/a | yes |
 
 ## Usage
 

main.tf

@@ -128,7 +128,7 @@ data "template_file" "shuntingyard_config_yaml" {
   vars {
     source_metastore_uri    = "${var.source_metastore_uri}"
     target_metastore_uri    = "${var.target_metastore_uri}"
-    shuntingyard_sqs_queue = "${aws_sqs_queue.shuntingyard_sqs_queue.id}"
+    shuntingyard_sqs_queue  = "${aws_sqs_queue.shuntingyard_sqs_queue.id}"
     selected_tables         = "${var.selected_tables}"
   }
 }
@@ -143,6 +143,7 @@ data "template_file" "shuntingyard" {
     region                   = "${var.aws_region}"
     loggroup                 = "${aws_cloudwatch_log_group.shuntingyard_ecs.name}"
     shuntingyard_config_yaml = "${base64encode(data.template_file.shuntingyard_config_yaml.rendered)}"
+    ct_common_config_yaml    = "${base64encode(var.ct_common_config_yaml)}"
   }
 }
 

templates/shuntingyard.json

@@ -19,6 +19,10 @@
      {
         "name": "SHUNTINGYARD_CONFIG_YAML",
         "value": "${shuntingyard_config_yaml}"
+     },
+     {
+        "name": "CT_COMMON_CONFIG_YAML",
+        "value": "${ct_common_config_yaml}"
      }
     ]
   }

variables.tf

@@ -68,8 +68,13 @@ variable "docker_version" {
   type        = "string"
 }
 
+variable "ct_common_config_yaml" {
+  description = "Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train."
+  type        = "string"
+}
+
 variable "allowed_s3_buckets" {
-  description = "List of S3 Buckets to which SY will have read-write access."
+  description = "List of S3 Buckets to which Shunting Yard will have read-write access."
   type        = "list"
 }
 
@@ -89,6 +94,6 @@ variable "metastore_events_sns_topic" {
 }
 
 variable "selected_tables" {
-  description = "Tables selected for Shunting Yard Replication."
+  description = "Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2`"
   type        = "string"
 }