Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add cluster replicas to Bigtable module. #1075

Merged
merged 4 commits into from
Dec 30, 2022
Merged

Add cluster replicas to Bigtable module. #1075

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2057939
Add cluster replicas to Bigtable module.
iht Dec 28, 2022
b095596
Fix TF code format
iht Dec 28, 2022
a733bfe
Use map for local clusters variable
iht Dec 30, 2022
4cccad9
Merge branch 'master' into bigtable_replication
iht Dec 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 84 additions & 42 deletions modules/bigtable-instance/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ This module allows managing a single BigTable instance, including access configu
## TODO

- [ ] support bigtable_app_profile
- [ ] support cluster replicas
- [ ] support IAM for tables

## Examples
Expand All @@ -18,8 +17,11 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-west1-b"
clusters = {
my-cluster = {
zone = "europe-west1-b"
}
}
tables = {
test1 = {},
test2 = {
Expand All @@ -41,8 +43,11 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-west1-b"
clusters = {
my-cluster = {
zone = "europe-west1-b"
}
}
tables = {
test1 = {},
test2 = {
Expand All @@ -63,6 +68,29 @@ module "bigtable-instance" {
# tftest modules=1 resources=4
```

### Instance with replication enabled

```hcl

module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
clusters = {
first-cluster = {
zone = "europe-west1-b"
}
second-cluster = {
zone = "europe-southwest1-a"
}
third-cluster = {
zone = "us-central1-b"
}
}
}
# tftest modules=1 resources=1
```

### Instance with garbage collection policy

```hcl
Expand All @@ -71,8 +99,11 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-west1-b"
clusters = {
my-cluster = {
zone = "europe-west1-b"
}
}
tables = {
test1 = {
column_families = {
Expand Down Expand Up @@ -102,8 +133,11 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-west1-b"
clusters = {
my-cluster = {
zone = "europe-west1-b"
}
}
default_gc_policy = {
deletion_policy = "ABANDON"
max_age = "18h"
Expand Down Expand Up @@ -131,9 +165,12 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-west1-b"
num_nodes = 5
clusters = {
my-cluster = {
zone = "europe-west1-b"
num_nodes = 5
}
}
}
# tftest modules=1 resources=1
```
Expand All @@ -148,13 +185,18 @@ module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-southwest1-b"
autoscaling_config = {
min_nodes = 3
max_nodes = 7
cpu_target = 70
clusters = {
my-cluster = {
zone = "europe-southwest1-b"
autoscaling = {
min_nodes = 3
max_nodes = 7
cpu_target = 70
}
}
}


}
# tftest modules=1 resources=1
```
Expand All @@ -164,17 +206,20 @@ module "bigtable-instance" {
```hcl

module "bigtable-instance" {
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
cluster_id = "instance"
zone = "europe-southwest1-a"
storage_type = "SSD"
autoscaling_config = {
min_nodes = 3
max_nodes = 7
cpu_target = 70
storage_target = 4096
source = "./fabric/modules/bigtable-instance"
project_id = "my-project"
name = "instance"
clusters = {
my-cluster = {
zone = "europe-southwest1-a"
storage_type = "SSD"
autoscaling = {
min_nodes = 3
max_nodes = 7
cpu_target = 70
storage_target = 4096
}
}
}
}
# tftest modules=1 resources=1
Expand All @@ -185,19 +230,16 @@ module "bigtable-instance" {

| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [name](variables.tf#L68) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L79) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
| [zone](variables.tf#L109) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object&#40;&#123;&#10; min_nodes &#61; number&#10; max_nodes &#61; number&#10; cpu_target &#61; number&#10; storage_target &#61; optional&#40;number, null&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object&#40;&#123;&#10; deletion_policy &#61; optional&#40;string&#41;&#10; gc_rules &#61; optional&#40;string&#41;&#10; mode &#61; optional&#40;string&#41;&#10; max_age &#61; optional&#40;string&#41;&#10; max_version &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
| [iam](variables.tf#L56) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [instance_type](variables.tf#L62) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
| [num_nodes](variables.tf#L73) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
| [storage_type](variables.tf#L84) | The storage type to use. | <code>string</code> | | <code>&#34;SSD&#34;</code> |
| [tables](variables.tf#L90) | Tables to be created in the BigTable instance. | <code title="map&#40;object&#40;&#123;&#10; split_keys &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; column_families &#61; optional&#40;map&#40;object&#40;&#10; &#123;&#10; gc_policy &#61; optional&#40;object&#40;&#123;&#10; deletion_policy &#61; optional&#40;string&#41;&#10; gc_rules &#61; optional&#40;string&#41;&#10; mode &#61; optional&#40;string&#41;&#10; max_age &#61; optional&#40;string&#41;&#10; max_version &#61; optional&#40;string&#41;&#10; &#125;&#41;, null&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | <code title="map&#40;object&#40;&#123;&#10; zone &#61; optional&#40;string&#41;&#10; storage_type &#61; optional&#40;string&#41;&#10; num_nodes &#61; optional&#40;number&#41;&#10; autoscaling &#61; optional&#40;object&#40;&#123;&#10; min_nodes &#61; number&#10; max_nodes &#61; number&#10; cpu_target &#61; number&#10; storage_target &#61; optional&#40;number&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ | |
| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | <code title="object&#40;&#123;&#10; min_nodes &#61; number&#10; max_nodes &#61; number&#10; cpu_target &#61; number&#10; storage_target &#61; optional&#40;number&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object&#40;&#123;&#10; deletion_policy &#61; optional&#40;string&#41;&#10; gc_rules &#61; optional&#40;string&#41;&#10; mode &#61; optional&#40;string&#41;&#10; max_age &#61; optional&#40;string&#41;&#10; max_version &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | <code title="map&#40;object&#40;&#123;&#10; split_keys &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; column_families &#61; optional&#40;map&#40;object&#40;&#10; &#123;&#10; gc_policy &#61; optional&#40;object&#40;&#123;&#10; deletion_policy &#61; optional&#40;string&#41;&#10; gc_rules &#61; optional&#40;string&#41;&#10; mode &#61; optional&#40;string&#41;&#10; max_age &#61; optional&#40;string&#41;&#10; max_version &#61; optional&#40;string&#41;&#10; &#125;&#41;, null&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |

## Outputs

Expand Down
48 changes: 31 additions & 17 deletions modules/bigtable-instance/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/

locals {
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
gc_pairs = flatten([
for table, table_obj in var.tables : [
for cf, cf_obj in table_obj.column_families : {
Expand All @@ -25,30 +24,45 @@ locals {
}
]
])

clusters_autoscaling = flatten([
iht marked this conversation as resolved.
Show resolved Hide resolved
for cluster, obj in var.clusters : [{
cluster_id = cluster
zone = obj.zone
storage_type = obj.storage_type
num_nodes = obj.autoscaling == null && var.default_autoscaling == null ? obj.num_nodes : null
autoscaling = obj.autoscaling == null ? var.default_autoscaling : obj.autoscaling
}]
])
}

resource "google_bigtable_instance" "default" {
project = var.project_id
name = var.name
cluster {
cluster_id = var.cluster_id
zone = var.zone
storage_type = var.storage_type
num_nodes = local.num_nodes
dynamic "autoscaling_config" {
for_each = var.autoscaling_config == null ? [] : [""]
content {
min_nodes = var.autoscaling_config.min_nodes
max_nodes = var.autoscaling_config.max_nodes
cpu_target = var.autoscaling_config.cpu_target
storage_target = var.autoscaling_config.storage_target
}
}
}
instance_type = var.instance_type

instance_type = var.instance_type
display_name = var.display_name == null ? var.display_name : var.name
deletion_protection = var.deletion_protection

dynamic "cluster" {
for_each = { for k, v in local.clusters_autoscaling : k => v }
content {
cluster_id = cluster.value.cluster_id
zone = cluster.value.zone
storage_type = cluster.value.storage_type
num_nodes = cluster.value.num_nodes

dynamic "autoscaling_config" {
for_each = cluster.value.autoscaling == null ? [] : [""]
content {
min_nodes = cluster.value.autoscaling.min_nodes
max_nodes = cluster.value.autoscaling.max_nodes
cpu_target = cluster.value.autoscaling.cpu_target
storage_target = cluster.value.autoscaling.storage_target
}
}
}
}
}

resource "google_bigtable_instance_iam_binding" "default" {
Expand Down
45 changes: 19 additions & 26 deletions modules/bigtable-instance/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,33 @@
* limitations under the License.
*/

variable "autoscaling_config" {
description = "Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored."
variable "clusters" {
description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored."
nullable = false
type = map(object({
zone = optional(string)
storage_type = optional(string)
num_nodes = optional(number)
autoscaling = optional(object({
min_nodes = number
max_nodes = number
cpu_target = number
storage_target = optional(number)
}))
}))
}

variable "default_autoscaling" {
description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details."
type = object({
min_nodes = number
max_nodes = number
cpu_target = number
storage_target = optional(number, null)
storage_target = optional(number)
})
default = null
}

variable "cluster_id" {
description = "The ID of the Cloud Bigtable cluster."
type = string
default = "europe-west1"
}

variable "default_gc_policy" {
description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
type = object({
Expand Down Expand Up @@ -70,23 +80,11 @@ variable "name" {
type = string
}

variable "num_nodes" {
description = "The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling."
type = number
default = 1
}

variable "project_id" {
description = "Id of the project where datasets will be created."
type = string
}

variable "storage_type" {
description = "The storage type to use."
type = string
default = "SSD"
}

variable "tables" {
description = "Tables to be created in the BigTable instance."
nullable = false
Expand All @@ -105,8 +103,3 @@ variable "tables" {
}))
default = {}
}

variable "zone" {
description = "The zone to create the Cloud Bigtable cluster in."
type = string
}
6 changes: 5 additions & 1 deletion tests/modules/bigtable_instance/fixture/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,9 @@ module "test" {
}

}
zone = var.zone
clusters = {
test = {
zone = var.zone
}
}
}