diff --git a/CHANGELOG.md b/CHANGELOG.md index 8699e32..e236c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.2.0] - TBD +### Added +- Support for Docker Auth. +- Ability to store credentials for Docker auth in AWS secrets manager. +- Cloudwatch dashboard. +- Cloudwatch alerts. +- SQS queue permissions. + +### Changed +- `selected_tables` variable is now a list. + ## [1.1.0] - 2019-02-08 ### Added - New variable for passing Circus Train common configurations to Shunting Yard. eg. Graphite. diff --git a/README.md b/README.md index 301e01c..94865c4 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Terraform module for setting up infrastructure for [Shunting Yard](https://github.com/HotelsDotCom/shunting-yard). -For more information please refer to the main [Apiary](https://github.com/ExpediaInc/apiary) project page. +For more information please refer to the main [Apiary](https://github.com/ExpediaGroup/apiary) project page. ## Variables | Name | Description | Type | Default | Required | @@ -14,13 +14,16 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | ct\_common\_config\_yaml | Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train. | string | n/a | yes | | docker\_image | Full path of Shunting Yard Docker image. | string | n/a | yes | | docker\_version | Shunting Yard Docker image version. | string | n/a | yes | +| docker\_registry\_auth\_secret\_name | Docker Registry authentication SecretManager secret name. | string | `` | no | | instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no | | memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no | | metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes | -| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | n/a | yes | +| shuntingyard\_sqs\_queue\_wait\_timeout | Shunting Yard SQS queue wait timeout (in seconds) | string | 15 | no | +| shuntingyard\_sqs\_queue\_stale\_messages\_timeout | Shunting Yard SQS queue stale messages alert timeout (in seconds) | string | 300 | no | +| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `[ "database_1.table_1", "database_2.table_2" ]` | list | [] | no | | source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes | | subnets | ECS container subnets. | list | n/a | yes | -| tags | A map of tags to apply to resources. | map | `` | no | +| shuntingyard\_tags | A map of tags to apply to resources. | map | `` | no | | target\_metastore\_uri | Target Metastore URI for Shunting Yard. | string | n/a | yes | | vpc\_id | VPC ID. | string | n/a | yes | @@ -29,26 +32,23 @@ For more information please refer to the main [Apiary](https://github.com/Expedi Example module invocation: ``` module "apiary-shuntingyard" { - source = "git::https://github.com/ExpediaInc/apiary-replication.git?ref=master" - instance_name = "shuntingyard-test" - aws_region = "us-west-2" - vpc_id = "vpc-1" - subnets = ["subnet-1", "subnet-2"] - allowed_s3_buckets = ["bucket-1", "bucket-2"] - - tags = { - Name = "Apiary-Shuntingyard" - Team = "Operations" - } - - source_metastore_uri = "thrift://ip-address:9083" - target_metastore_uri = "thrift://ip-address:9083" - metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" - selected_tables = "database_1.table_1, database_2.table_2" - ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" - - docker_image = "your.docker.repo/apiary-shuntingyard" - docker_version = "latest" + source = "git::https://github.com/ExpediaGroup/apiary-replication.git" + aws_region = "us-west-2" + vpc_id = "vpc-1" + subnets = ["subnet-1", "subnet-2"] + instance_name = "shuntingyard-test" + docker_image = "your.docker.repo/apiary-shuntingyard" + docker_version = "latest" + ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" + source_metastore_uri = "thrift://ip-address:9083" + target_metastore_uri = "thrift://ip-address:9083" + metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" + selected_tables = [ "database_1.table_1", "database_2.table_2" ] + allowed_s3_buckets = [ "bucket-1", "bucket-2" ] + shuntingyard_tags = { + Name = "Apiary Replication" + Team = "Operations" + } } ``` diff --git a/cloudwatch.tf b/cloudwatch.tf new file mode 100644 index 0000000..ccd397a --- /dev/null +++ b/cloudwatch.tf @@ -0,0 +1,169 @@ +/** + * Copyright (C) 2019 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + */ + +data "template_file" "ecs_widgets" { + template = <