From a3d541a9fa5b83ef05f0375cc3a62ad476865fc9 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Tue, 11 Jun 2019 15:17:42 -0700 Subject: [PATCH 01/17] Adding database_encryption option --- .gitignore | 1 + README.md | 1 + autogen/cluster_regional.tf | 3 ++- autogen/cluster_zonal.tf | 3 ++- autogen/variables.tf | 12 ++++++++++++ cluster_regional.tf | 3 ++- cluster_zonal.tf | 3 ++- modules/private-cluster/README.md | 1 + modules/private-cluster/cluster_regional.tf | 3 ++- modules/private-cluster/cluster_zonal.tf | 3 ++- modules/private-cluster/variables.tf | 13 +++++++++++++ variables.tf | 12 ++++++++++++ 12 files changed, 52 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 899be3e749..63e84ba6f7 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ credentials.json # File to populate env vars used by Docker test runs .envrc +marko-env diff --git a/README.md b/README.md index e56d7db123..7be3665ed5 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o |------|-------------|:----:|:-----:|:-----:| | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| database\_encryption | Application-layer Secrets Encryption settings. Example: database_encryption = [{ state = "ENCRYPTED", key_name = "projects/my-project/locations/global/keyRings/my-ring/cryptoKeys/my-key" }] | list | `` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 2c5d223014..ae3677e5ee 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "primary" { description = "${var.description}" project = "${var.project_id}" - region = "${var.region}" + location = "${var.region}" node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -121,6 +121,7 @@ resource "google_container_cluster" "primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = "${var.database_encryption}" } /****************************************** diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 81d371caa4..cad750bad5 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "zonal_primary" { description = "${var.description}" project = "${var.project_id}" - zone = "${var.zones[0]}" + location = "${var.zones[0]}" node_locations = ["${slice(var.zones,1,length(var.zones))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -121,6 +121,7 @@ resource "google_container_cluster" "zonal_primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = "${var.database_encryption}" } /****************************************** diff --git a/autogen/variables.tf b/autogen/variables.tf index dc2b86021a..2a31b2abf8 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -300,3 +300,15 @@ variable "issue_client_certificate" { description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" default = "false" } + +variable "database_encryption" { + description = <` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 5c0818a904..9941e91d89 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "primary" { description = "${var.description}" project = "${var.project_id}" - region = "${var.region}" + location = "${var.region}" node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -108,6 +108,7 @@ resource "google_container_cluster" "primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = "${var.database_encryption}" } /****************************************** diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index 39fbcb6098..9246c4108a 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "zonal_primary" { description = "${var.description}" project = "${var.project_id}" - zone = "${var.zones[0]}" + location = "${var.zones[0]}" node_locations = ["${slice(var.zones,1,length(var.zones))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -108,6 +108,7 @@ resource "google_container_cluster" "zonal_primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = "${var.database_encryption}" } /****************************************** diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 9d6b123ac8..373aa524b2 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -284,3 +284,16 @@ variable "issue_client_certificate" { description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" default = "false" } + +variable "database_encryption" { + description = < Date: Tue, 11 Jun 2019 15:23:08 -0700 Subject: [PATCH 02/17] setting only google-beta provider --- cluster_regional.tf | 2 +- cluster_zonal.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster_regional.tf b/cluster_regional.tf index e91f47f108..0ea7d235c7 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -20,7 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google" + provider = "google-beta" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf index a3996e4452..8134d6fbc4 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -20,7 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google" + provider = "google-beta" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" From 34e9e3c21d09c479b5b11eff5076bd9b6c6bd098 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Wed, 12 Jun 2019 10:54:54 -0700 Subject: [PATCH 03/17] Changing depricated fields zone and region to location --- autogen/cluster_regional.tf | 2 +- autogen/cluster_zonal.tf | 2 +- autogen/main.tf | 4 ++-- cluster_regional.tf | 2 +- cluster_zonal.tf | 2 +- main.tf | 2 +- modules/private-cluster/cluster_regional.tf | 2 +- modules/private-cluster/cluster_zonal.tf | 2 +- modules/private-cluster/main.tf | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index ae3677e5ee..c9262f4a1f 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -132,7 +132,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - region = "${var.region}" + location = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index cad750bad5..f0d135a655 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -132,7 +132,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - zone = "${var.zones[0]}" + location = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/autogen/main.tf b/autogen/main.tf index ba8bb1c329..552a142a77 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -198,7 +198,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - region = "${var.region}" + location = "${var.region}" project = "${var.project_id}" } @@ -207,7 +207,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + location = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" project = "${var.project_id}" } diff --git a/cluster_regional.tf b/cluster_regional.tf index 0ea7d235c7..da7cf4a369 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -112,7 +112,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - region = "${var.region}" + location = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 8134d6fbc4..775e692e29 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -112,7 +112,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - zone = "${var.zones[0]}" + location = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/main.tf b/main.tf index e96d301c43..cd4f7bd2df 100644 --- a/main.tf +++ b/main.tf @@ -152,7 +152,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - region = "${var.region}" + location = "${var.region}" project = "${var.project_id}" } diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 9941e91d89..166128be7a 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -119,7 +119,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - region = "${var.region}" + location = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index 9246c4108a..62722b60c1 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -119,7 +119,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - zone = "${var.zones[0]}" + location = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 95e28ae4b3..1d80798166 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -167,7 +167,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - region = "${var.region}" + location = "${var.region}" project = "${var.project_id}" } From 57305fe1a2bf3ea313479eb36babe6d533ca9e2f Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Wed, 12 Jun 2019 13:08:22 -0700 Subject: [PATCH 04/17] Updating provider in tests --- examples/simple_regional_private/main.tf | 5 +++++ examples/simple_zonal_private/main.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index 29ffe9528e..60e9c34b3f 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -18,6 +18,11 @@ locals { cluster_type = "simple-regional-private" } +provider "google" { + version = "~> 2.7.0" + region = "${var.region}" +} + provider "google-beta" { version = "~> 2.7.0" region = "${var.region}" diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index e5d643e9d3..d9e67f8193 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -18,6 +18,11 @@ locals { cluster_type = "simple-regional-private" } +provider "google" { + version = "~> 2.7.0" + region = "${var.region}" +} + provider "google-beta" { version = "~> 2.7.0" region = "${var.region}" From a4207620e38dd3a8a9b56e7b5f83aaba159b3c30 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Wed, 12 Jun 2019 14:05:23 -0700 Subject: [PATCH 05/17] Revert .gitignore changes --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 63e84ba6f7..79f3c01523 100644 --- a/.gitignore +++ b/.gitignore @@ -49,5 +49,4 @@ test/integration/tmp credentials.json # File to populate env vars used by Docker test runs -.envrc -marko-env +.envrc \ No newline at end of file From ed1370ffbed5cd46963ba0c268cb81097d6b14ec Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Wed, 12 Jun 2019 14:41:07 -0700 Subject: [PATCH 06/17] Reverting changes to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 79f3c01523..899be3e749 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,4 @@ test/integration/tmp credentials.json # File to populate env vars used by Docker test runs -.envrc \ No newline at end of file +.envrc From 30970a9bf3535ec1a7669efa445b5c805f12177f Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Wed, 12 Jun 2019 15:27:24 -0700 Subject: [PATCH 07/17] Fixing failing test 'kubernetes nodes pool-02' --- test/integration/node_pool/controls/kubectl.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb index 8804d6e83e..762046b6d1 100644 --- a/test/integration/node_pool/controls/kubectl.rb +++ b/test/integration/node_pool/controls/kubectl.rb @@ -78,7 +78,7 @@ effect: "PreferNoSchedule", key: "all-pools-example", value: "true", - } + }, ]) end end From 09093e25fbcd1e7ef7b2357597fe7613fe1b1e72 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Thu, 13 Jun 2019 09:31:56 -0700 Subject: [PATCH 08/17] Reverting location back to zone and regions. Reverting node_pool test case. Updating documentation --- README.md | 1 + autogen/README.md | 2 -- autogen/cluster_regional.tf | 4 ++-- autogen/cluster_zonal.tf | 4 ++-- autogen/main.tf | 4 ++-- cluster_regional.tf | 4 ++-- cluster_zonal.tf | 4 ++-- examples/simple_regional_private/main.tf | 5 ----- examples/simple_zonal_private/main.tf | 5 ----- main.tf | 2 +- modules/private-cluster/README.md | 1 + modules/private-cluster/cluster_regional.tf | 4 ++-- modules/private-cluster/cluster_zonal.tf | 4 ++-- modules/private-cluster/main.tf | 2 +- test/integration/node_pool/controls/kubectl.rb | 5 +++++ 15 files changed, 23 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7be3665ed5..9c6033bb43 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 ### Configure a Service Account diff --git a/autogen/README.md b/autogen/README.md index 77c45feee6..a5a36b4a35 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -143,9 +143,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [Terraform](https://www.terraform.io/downloads.html) 0.11.x {% if private_cluster or beta_cluster %} - [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 -{% else %} - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 -{% endif %} ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index c9262f4a1f..4890dfd1c8 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -132,7 +132,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index f0d135a655..56b8a4b476 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "zonal_primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" node_locations = ["${slice(var.zones,1,length(var.zones))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -132,7 +132,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/autogen/main.tf b/autogen/main.tf index 552a142a77..ba8bb1c329 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -198,7 +198,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - location = "${var.region}" + region = "${var.region}" project = "${var.project_id}" } @@ -207,7 +207,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - location = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" project = "${var.project_id}" } diff --git a/cluster_regional.tf b/cluster_regional.tf index da7cf4a369..8d02d2ff39 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -112,7 +112,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 775e692e29..266f648fcf 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "zonal_primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" node_locations = ["${slice(var.zones,1,length(var.zones))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -112,7 +112,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index 60e9c34b3f..29ffe9528e 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -18,11 +18,6 @@ locals { cluster_type = "simple-regional-private" } -provider "google" { - version = "~> 2.7.0" - region = "${var.region}" -} - provider "google-beta" { version = "~> 2.7.0" region = "${var.region}" diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index d9e67f8193..e5d643e9d3 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -18,11 +18,6 @@ locals { cluster_type = "simple-regional-private" } -provider "google" { - version = "~> 2.7.0" - region = "${var.region}" -} - provider "google-beta" { version = "~> 2.7.0" region = "${var.region}" diff --git a/main.tf b/main.tf index cd4f7bd2df..e96d301c43 100644 --- a/main.tf +++ b/main.tf @@ -152,7 +152,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - location = "${var.region}" + region = "${var.region}" project = "${var.project_id}" } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index e97b9be733..c2a881c5d6 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -210,6 +210,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x - [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 166128be7a..02dcfb4a3b 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -119,7 +119,7 @@ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.region}" + region = "${var.region}" cluster = "${google_container_cluster.primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index 62722b60c1..d8bbe9fa2f 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -26,7 +26,7 @@ resource "google_container_cluster" "zonal_primary" { description = "${var.description}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" node_locations = ["${slice(var.zones,1,length(var.zones))}"] network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" @@ -119,7 +119,7 @@ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" - location = "${var.zones[0]}" + zone = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 1d80798166..95e28ae4b3 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -167,7 +167,7 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - location = "${var.region}" + region = "${var.region}" project = "${var.project_id}" } diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb index 762046b6d1..e14852f632 100644 --- a/test/integration/node_pool/controls/kubectl.rb +++ b/test/integration/node_pool/controls/kubectl.rb @@ -79,6 +79,11 @@ key: "all-pools-example", value: "true", }, + { + effect: "NoSchedule", + key: "nvidia.com/gpu", + value: "present", + }, ]) end end From 91185c01b23b3595c55bb6881668523a838c91ea Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Fri, 14 Jun 2019 09:18:18 -0700 Subject: [PATCH 09/17] Adding integration tests for database_encryption --- .kitchen.yml | 9 +++ examples/database_encryption/README.md | 47 ++++++++++++++ examples/database_encryption/main.tf | 46 ++++++++++++++ examples/database_encryption/outputs.tf | 34 ++++++++++ examples/database_encryption/test_outputs.tf | 63 +++++++++++++++++++ examples/database_encryption/variables.tf | 54 ++++++++++++++++ test/ci/database-encryption.yml | 18 ++++++ test/fixtures/database_encryption/example.tf | 32 ++++++++++ test/fixtures/database_encryption/kms.tf | 40 ++++++++++++ test/fixtures/database_encryption/network.tf | 52 +++++++++++++++ test/fixtures/database_encryption/outputs.tf | 1 + .../database_encryption/terraform.tfvars | 1 + .../fixtures/database_encryption/variables.tf | 1 + test/fixtures/shared/variables.tf | 6 ++ .../database_encryption/controls/gcloud.rb | 44 +++++++++++++ .../database_encryption/inspec.yml | 18 ++++++ 16 files changed, 466 insertions(+) create mode 100644 examples/database_encryption/README.md create mode 100644 examples/database_encryption/main.tf create mode 100644 examples/database_encryption/outputs.tf create mode 100644 examples/database_encryption/test_outputs.tf create mode 100644 examples/database_encryption/variables.tf create mode 100644 test/ci/database-encryption.yml create mode 100644 test/fixtures/database_encryption/example.tf create mode 100644 test/fixtures/database_encryption/kms.tf create mode 100644 test/fixtures/database_encryption/network.tf create mode 120000 test/fixtures/database_encryption/outputs.tf create mode 120000 test/fixtures/database_encryption/terraform.tfvars create mode 120000 test/fixtures/database_encryption/variables.tf create mode 100644 test/integration/database_encryption/controls/gcloud.rb create mode 100644 test/integration/database_encryption/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 7c3e2fea07..ffb7706cba 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -104,3 +104,12 @@ suites: systems: - name: stub_domains_private backend: local + - name: "database_encryption" + driver: + root_module_directory: test/fixtures/database_encryption + verifier: + systems: + - name: gcloud + backend: local + controls: + - gcloud \ No newline at end of file diff --git a/examples/database_encryption/README.md b/examples/database_encryption/README.md new file mode 100644 index 0000000000..f10919d50c --- /dev/null +++ b/examples/database_encryption/README.md @@ -0,0 +1,47 @@ +# Database Encryption + +This example illustrates how to create a simple cluster with database encryption described in +https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| database\_encryption | Database encryption key and state | list | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/database_encryption/main.tf b/examples/database_encryption/main.tf new file mode 100644 index 0000000000..5bdc49a688 --- /dev/null +++ b/examples/database_encryption/main.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "database-encryption" +} + +provider "google" { + version = "~> 2.7.0" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.7.0" + region = "${var.region}" +} + +module "gke" { + source = "../../" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = false + region = "${var.region}" + zones = "${var.zones}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "create" + database_encryption = "${var.database_encryption}" +} + +data "google_client_config" "default" {} diff --git a/examples/database_encryption/outputs.tf b/examples/database_encryption/outputs.tf new file mode 100644 index 0000000000..9881585a48 --- /dev/null +++ b/examples/database_encryption/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + value = "${module.gke.ca_certificate}" +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/database_encryption/test_outputs.tf b/examples/database_encryption/test_outputs.tf new file mode 100644 index 0000000000..c1d4352219 --- /dev/null +++ b/examples/database_encryption/test_outputs.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${module.gke.region}" +} + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "location" { + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = "${module.gke.zones}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/database_encryption/variables.tf b/examples/database_encryption/variables.tf new file mode 100644 index 0000000000..72bf8a59ec --- /dev/null +++ b/examples/database_encryption/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "zones" { + type = "list" + description = "The zone to host the cluster in (required if is a zonal cluster)" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "database_encryption" { + type = "list" + description = "Database encryption key and state" +} \ No newline at end of file diff --git a/test/ci/database-encryption.yml b/test/ci/database-encryption.yml new file mode 100644 index 0000000000..acb2acd8e4 --- /dev/null +++ b/test/ci/database-encryption.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "database-encryption-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/database_encryption/example.tf b/test/fixtures/database_encryption/example.tf new file mode 100644 index 0000000000..5cd1bb9e12 --- /dev/null +++ b/test/fixtures/database_encryption/example.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/database_encryption" + + project_id = "${var.project_id}" + cluster_name_suffix = "-${random_string.suffix.result}" + region = "${var.region}" + zones = ["${slice(var.zones,0,1)}"] + network = "${google_compute_network.main.name}" + subnetwork = "${google_compute_subnetwork.main.name}" + ip_range_pods = "${google_compute_subnetwork.main.secondary_ip_range.0.range_name}" + ip_range_services = "${google_compute_subnetwork.main.secondary_ip_range.1.range_name}" + database_encryption = [{ + state = "ENCRYPTED" + key_name = "projects/${var.project_id}/locations/${var.region}/keyRings/db-key-ring/cryptoKeys/db-key" + }] +} \ No newline at end of file diff --git a/test/fixtures/database_encryption/kms.tf b/test/fixtures/database_encryption/kms.tf new file mode 100644 index 0000000000..4b814c51d0 --- /dev/null +++ b/test/fixtures/database_encryption/kms.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_kms_key_ring" "db_key_ring" { + name = "db-key-ring" + project = "${var.project_id}" + location = "${var.region}" +} + +resource "google_kms_crypto_key" "db_key" { + name = "db-key" + key_ring = "${google_kms_key_ring.db_key_ring.self_link}" + + version_template { + algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" + } +} + +data "google_project" "project_data" { + project_id = "${var.project_id}" +} + +resource "google_kms_crypto_key_iam_member" "kms_permissions" { + crypto_key_id = "${google_kms_crypto_key.db_key.id}" + member = "serviceAccount:service-${data.google_project.project_data.number}@container-engine-robot.iam.gserviceaccount.com" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" +} \ No newline at end of file diff --git a/test/fixtures/database_encryption/network.tf b/test/fixtures/database_encryption/network.tf new file mode 100644 index 0000000000..b96a5fb2dc --- /dev/null +++ b/test/fixtures/database_encryption/network.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = "${var.project_id}" +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = "false" + depends_on = [ + "google_kms_crypto_key.db_key", + "google_kms_crypto_key_iam_member.kms_permissions", + "google_kms_key_ring.db_key_ring", + ] +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "${var.region}" + network = "${google_compute_network.main.self_link}" + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} diff --git a/test/fixtures/database_encryption/outputs.tf b/test/fixtures/database_encryption/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/database_encryption/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/database_encryption/terraform.tfvars b/test/fixtures/database_encryption/terraform.tfvars new file mode 120000 index 0000000000..08ac6f4724 --- /dev/null +++ b/test/fixtures/database_encryption/terraform.tfvars @@ -0,0 +1 @@ +../shared/terraform.tfvars \ No newline at end of file diff --git a/test/fixtures/database_encryption/variables.tf b/test/fixtures/database_encryption/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/database_encryption/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index 28b827b0d5..cb37a78ab1 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -31,3 +31,9 @@ variable "zones" { variable "compute_engine_service_account" { description = "The email address of the service account to associate with the GKE cluster" } + +variable "database_encryption" { + description = "Database encryption key and state" + type = "list" + default = [] +} \ No newline at end of file diff --git a/test/integration/database_encryption/controls/gcloud.rb b/test/integration/database_encryption/controls/gcloud.rb new file mode 100644 index 0000000000..3abd1ee511 --- /dev/null +++ b/test/integration/database_encryption/controls/gcloud.rb @@ -0,0 +1,44 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +region = attribute('region') + +control "gcloud" do + title "GKE " + describe command("gcloud beta --project=#{project_id} container clusters --zone #{location} describe #{cluster_name} --format 'json(databaseEncryption)'") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout)["databaseEncryption"] + else + {} + end + end + + describe "database-encryption" do + it "is ENCRYPTED" do + expect(data['state']).to eq "ENCRYPTED" + end + + it "has valid key" do + expect(data['keyName']).to eq "projects/#{project_id}/locations/#{region}/keyRings/db-key-ring/cryptoKeys/db-key" + end + end + end +end diff --git a/test/integration/database_encryption/inspec.yml b/test/integration/database_encryption/inspec.yml new file mode 100644 index 0000000000..4a56322d7e --- /dev/null +++ b/test/integration/database_encryption/inspec.yml @@ -0,0 +1,18 @@ +name: simple_zonal +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: region + required: true + type: string + - name: cluster_name + required: true + type: string \ No newline at end of file From e9e1e3f7efec2a83d313cf430fa39a2d754c8ba6 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Fri, 14 Jun 2019 09:29:52 -0700 Subject: [PATCH 10/17] Fixing trailing white spaces --- examples/database_encryption/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/database_encryption/README.md b/examples/database_encryption/README.md index f10919d50c..3a694e2e8e 100644 --- a/examples/database_encryption/README.md +++ b/examples/database_encryption/README.md @@ -1,6 +1,6 @@ # Database Encryption -This example illustrates how to create a simple cluster with database encryption described in +This example illustrates how to create a simple cluster with database encryption described in https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets [^]: (autogen_docs_start) From 49eea8500eafd1d57ed3a86ef0218fb54e8e03b1 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Mon, 17 Jun 2019 17:08:13 -0700 Subject: [PATCH 11/17] Fixing default variables for database_encryption to prevent node pool recreation --- autogen/variables.tf | 5 ++++- modules/private-cluster/variables.tf | 8 ++++++-- variables.tf | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/autogen/variables.tf b/autogen/variables.tf index 2a31b2abf8..634e49e811 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -310,5 +310,8 @@ variable "database_encryption" { }] EOF type = "list" - default = [] + default = [{ + state = "DECRYPTED" + key_name = "" + }] } \ No newline at end of file diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 373aa524b2..af7592561d 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -294,6 +294,10 @@ variable "database_encryption" { }] EOF - type = "list" - default = [] + type = "list" + + default = [{ + state = "DECRYPTED" + key_name = "" + }] } diff --git a/variables.tf b/variables.tf index e09b3b2b61..a2e34e305a 100644 --- a/variables.tf +++ b/variables.tf @@ -263,5 +263,8 @@ variable "database_encryption" { }] EOF type = "list" - default = [] + default = [{ + state = "DECRYPTED" + key_name = "" + }] } \ No newline at end of file From 22833824b7c01734fc9c56ac3e198bd4c039d8e4 Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Tue, 18 Jun 2019 10:17:36 -0700 Subject: [PATCH 12/17] Removing database_encryption tests --- .kitchen.yml | 11 +--- examples/database_encryption/README.md | 47 -------------- examples/database_encryption/main.tf | 46 -------------- examples/database_encryption/outputs.tf | 34 ---------- examples/database_encryption/test_outputs.tf | 63 ------------------- examples/database_encryption/variables.tf | 54 ---------------- test/fixtures/database_encryption/example.tf | 32 ---------- test/fixtures/database_encryption/kms.tf | 40 ------------ test/fixtures/database_encryption/network.tf | 52 --------------- test/fixtures/database_encryption/outputs.tf | 1 - .../database_encryption/terraform.tfvars | 1 - .../fixtures/database_encryption/variables.tf | 1 - 12 files changed, 1 insertion(+), 381 deletions(-) delete mode 100644 examples/database_encryption/README.md delete mode 100644 examples/database_encryption/main.tf delete mode 100644 examples/database_encryption/outputs.tf delete mode 100644 examples/database_encryption/test_outputs.tf delete mode 100644 examples/database_encryption/variables.tf delete mode 100644 test/fixtures/database_encryption/example.tf delete mode 100644 test/fixtures/database_encryption/kms.tf delete mode 100644 test/fixtures/database_encryption/network.tf delete mode 120000 test/fixtures/database_encryption/outputs.tf delete mode 120000 test/fixtures/database_encryption/terraform.tfvars delete mode 120000 test/fixtures/database_encryption/variables.tf diff --git a/.kitchen.yml b/.kitchen.yml index ffb7706cba..e723a711de 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -103,13 +103,4 @@ suites: root_module_directory: test/fixtures/stub_domains_private systems: - name: stub_domains_private - backend: local - - name: "database_encryption" - driver: - root_module_directory: test/fixtures/database_encryption - verifier: - systems: - - name: gcloud - backend: local - controls: - - gcloud \ No newline at end of file + backend: local \ No newline at end of file diff --git a/examples/database_encryption/README.md b/examples/database_encryption/README.md deleted file mode 100644 index 3a694e2e8e..0000000000 --- a/examples/database_encryption/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Database Encryption - -This example illustrates how to create a simple cluster with database encryption described in -https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets - -[^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | -| database\_encryption | Database encryption key and state | list | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | -| network | The VPC network to host the cluster in | string | n/a | yes | -| project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | -| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | | -| client\_token | | -| cluster\_name | Cluster name | -| ip\_range\_pods | The secondary IP range used for pods | -| ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | | -| location | | -| master\_kubernetes\_version | The master Kubernetes version | -| network | | -| project\_id | | -| region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| subnetwork | | -| zones | List of zones in which the cluster resides | - -[^]: (autogen_docs_end) - -To provision this example, run the following from within this directory: -- `terraform init` to get the plugins -- `terraform plan` to see the infrastructure plan -- `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure diff --git a/examples/database_encryption/main.tf b/examples/database_encryption/main.tf deleted file mode 100644 index 5bdc49a688..0000000000 --- a/examples/database_encryption/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_type = "database-encryption" -} - -provider "google" { - version = "~> 2.7.0" - region = "${var.region}" -} - -provider "google-beta" { - version = "~> 2.7.0" - region = "${var.region}" -} - -module "gke" { - source = "../../" - project_id = "${var.project_id}" - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = false - region = "${var.region}" - zones = "${var.zones}" - network = "${var.network}" - subnetwork = "${var.subnetwork}" - ip_range_pods = "${var.ip_range_pods}" - ip_range_services = "${var.ip_range_services}" - service_account = "create" - database_encryption = "${var.database_encryption}" -} - -data "google_client_config" "default" {} diff --git a/examples/database_encryption/outputs.tf b/examples/database_encryption/outputs.tf deleted file mode 100644 index 9881585a48..0000000000 --- a/examples/database_encryption/outputs.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "kubernetes_endpoint" { - sensitive = true - value = "${module.gke.endpoint}" -} - -output "client_token" { - sensitive = true - value = "${base64encode(data.google_client_config.default.access_token)}" -} - -output "ca_certificate" { - value = "${module.gke.ca_certificate}" -} - -output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." - value = "${module.gke.service_account}" -} diff --git a/examples/database_encryption/test_outputs.tf b/examples/database_encryption/test_outputs.tf deleted file mode 100644 index c1d4352219..0000000000 --- a/examples/database_encryption/test_outputs.tf +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// These outputs are used to test the module with kitchen-terraform -// They do not need to be included in real-world uses of this module - -output "project_id" { - value = "${var.project_id}" -} - -output "region" { - value = "${module.gke.region}" -} - -output "cluster_name" { - description = "Cluster name" - value = "${module.gke.name}" -} - -output "network" { - value = "${var.network}" -} - -output "subnetwork" { - value = "${var.subnetwork}" -} - -output "location" { - value = "${module.gke.location}" -} - -output "ip_range_pods" { - description = "The secondary IP range used for pods" - value = "${var.ip_range_pods}" -} - -output "ip_range_services" { - description = "The secondary IP range used for services" - value = "${var.ip_range_services}" -} - -output "zones" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" -} - -output "master_kubernetes_version" { - description = "The master Kubernetes version" - value = "${module.gke.master_version}" -} diff --git a/examples/database_encryption/variables.tf b/examples/database_encryption/variables.tf deleted file mode 100644 index 72bf8a59ec..0000000000 --- a/examples/database_encryption/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID to host the cluster in" -} - -variable "cluster_name_suffix" { - description = "A suffix to append to the default cluster name" - default = "" -} - -variable "region" { - description = "The region to host the cluster in" -} - -variable "zones" { - type = "list" - description = "The zone to host the cluster in (required if is a zonal cluster)" -} - -variable "network" { - description = "The VPC network to host the cluster in" -} - -variable "subnetwork" { - description = "The subnetwork to host the cluster in" -} - -variable "ip_range_pods" { - description = "The secondary ip range to use for pods" -} - -variable "ip_range_services" { - description = "The secondary ip range to use for pods" -} - -variable "database_encryption" { - type = "list" - description = "Database encryption key and state" -} \ No newline at end of file diff --git a/test/fixtures/database_encryption/example.tf b/test/fixtures/database_encryption/example.tf deleted file mode 100644 index 5cd1bb9e12..0000000000 --- a/test/fixtures/database_encryption/example.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "example" { - source = "../../../examples/database_encryption" - - project_id = "${var.project_id}" - cluster_name_suffix = "-${random_string.suffix.result}" - region = "${var.region}" - zones = ["${slice(var.zones,0,1)}"] - network = "${google_compute_network.main.name}" - subnetwork = "${google_compute_subnetwork.main.name}" - ip_range_pods = "${google_compute_subnetwork.main.secondary_ip_range.0.range_name}" - ip_range_services = "${google_compute_subnetwork.main.secondary_ip_range.1.range_name}" - database_encryption = [{ - state = "ENCRYPTED" - key_name = "projects/${var.project_id}/locations/${var.region}/keyRings/db-key-ring/cryptoKeys/db-key" - }] -} \ No newline at end of file diff --git a/test/fixtures/database_encryption/kms.tf b/test/fixtures/database_encryption/kms.tf deleted file mode 100644 index 4b814c51d0..0000000000 --- a/test/fixtures/database_encryption/kms.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_kms_key_ring" "db_key_ring" { - name = "db-key-ring" - project = "${var.project_id}" - location = "${var.region}" -} - -resource "google_kms_crypto_key" "db_key" { - name = "db-key" - key_ring = "${google_kms_key_ring.db_key_ring.self_link}" - - version_template { - algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" - } -} - -data "google_project" "project_data" { - project_id = "${var.project_id}" -} - -resource "google_kms_crypto_key_iam_member" "kms_permissions" { - crypto_key_id = "${google_kms_crypto_key.db_key.id}" - member = "serviceAccount:service-${data.google_project.project_data.number}@container-engine-robot.iam.gserviceaccount.com" - role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" -} \ No newline at end of file diff --git a/test/fixtures/database_encryption/network.tf b/test/fixtures/database_encryption/network.tf deleted file mode 100644 index b96a5fb2dc..0000000000 --- a/test/fixtures/database_encryption/network.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} - -provider "google" { - project = "${var.project_id}" -} - -resource "google_compute_network" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - auto_create_subnetworks = "false" - depends_on = [ - "google_kms_crypto_key.db_key", - "google_kms_crypto_key_iam_member.kms_permissions", - "google_kms_key_ring.db_key_ring", - ] -} - -resource "google_compute_subnetwork" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = "${var.region}" - network = "${google_compute_network.main.self_link}" - - secondary_ip_range { - range_name = "cft-gke-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "cft-gke-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} diff --git a/test/fixtures/database_encryption/outputs.tf b/test/fixtures/database_encryption/outputs.tf deleted file mode 120000 index 726bdc722f..0000000000 --- a/test/fixtures/database_encryption/outputs.tf +++ /dev/null @@ -1 +0,0 @@ -../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/database_encryption/terraform.tfvars b/test/fixtures/database_encryption/terraform.tfvars deleted file mode 120000 index 08ac6f4724..0000000000 --- a/test/fixtures/database_encryption/terraform.tfvars +++ /dev/null @@ -1 +0,0 @@ -../shared/terraform.tfvars \ No newline at end of file diff --git a/test/fixtures/database_encryption/variables.tf b/test/fixtures/database_encryption/variables.tf deleted file mode 120000 index c113c00a3d..0000000000 --- a/test/fixtures/database_encryption/variables.tf +++ /dev/null @@ -1 +0,0 @@ -../shared/variables.tf \ No newline at end of file From 81b7943272fdc6b98c3ba99be6da330c53b9ce9d Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Tue, 18 Jun 2019 10:30:03 -0700 Subject: [PATCH 13/17] Removing remaining tests and updating database_encryption wrapper --- autogen/cluster_regional.tf | 2 +- autogen/cluster_zonal.tf | 2 +- cluster_regional.tf | 2 +- cluster_zonal.tf | 2 +- modules/private-cluster/cluster_regional.tf | 2 +- modules/private-cluster/cluster_zonal.tf | 2 +- test/ci/database-encryption.yml | 18 -------- test/fixtures/shared/variables.tf | 6 --- .../database_encryption/controls/gcloud.rb | 44 ------------------- .../database_encryption/inspec.yml | 18 -------- 10 files changed, 6 insertions(+), 92 deletions(-) delete mode 100644 test/ci/database-encryption.yml delete mode 100644 test/integration/database_encryption/controls/gcloud.rb delete mode 100644 test/integration/database_encryption/inspec.yml diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 4890dfd1c8..c26191cbcc 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -121,7 +121,7 @@ resource "google_container_cluster" "primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 56b8a4b476..6c7778a92e 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -121,7 +121,7 @@ resource "google_container_cluster" "zonal_primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/cluster_regional.tf b/cluster_regional.tf index 8d02d2ff39..0fa8ef1508 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -101,7 +101,7 @@ resource "google_container_cluster" "primary" { } } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 266f648fcf..b58be37e93 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -101,7 +101,7 @@ resource "google_container_cluster" "zonal_primary" { } } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 02dcfb4a3b..beffdc8d4a 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -108,7 +108,7 @@ resource "google_container_cluster" "primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index d8bbe9fa2f..b36c6dd66f 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -108,7 +108,7 @@ resource "google_container_cluster" "zonal_primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = "${var.database_encryption}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/test/ci/database-encryption.yml b/test/ci/database-encryption.yml deleted file mode 100644 index acb2acd8e4..0000000000 --- a/test/ci/database-encryption.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - -platform: linux - -inputs: -- name: pull-request - path: terraform-google-kubernetes-engine - -run: - path: make - args: ['test_integration'] - dir: terraform-google-kubernetes-engine - -params: - SUITE: "database-encryption-local" - COMPUTE_ENGINE_SERVICE_ACCOUNT: "" - REGION: "us-east4" - ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index cb37a78ab1..28b827b0d5 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -31,9 +31,3 @@ variable "zones" { variable "compute_engine_service_account" { description = "The email address of the service account to associate with the GKE cluster" } - -variable "database_encryption" { - description = "Database encryption key and state" - type = "list" - default = [] -} \ No newline at end of file diff --git a/test/integration/database_encryption/controls/gcloud.rb b/test/integration/database_encryption/controls/gcloud.rb deleted file mode 100644 index 3abd1ee511..0000000000 --- a/test/integration/database_encryption/controls/gcloud.rb +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -project_id = attribute('project_id') -location = attribute('location') -cluster_name = attribute('cluster_name') -region = attribute('region') - -control "gcloud" do - title "GKE " - describe command("gcloud beta --project=#{project_id} container clusters --zone #{location} describe #{cluster_name} --format 'json(databaseEncryption)'") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout)["databaseEncryption"] - else - {} - end - end - - describe "database-encryption" do - it "is ENCRYPTED" do - expect(data['state']).to eq "ENCRYPTED" - end - - it "has valid key" do - expect(data['keyName']).to eq "projects/#{project_id}/locations/#{region}/keyRings/db-key-ring/cryptoKeys/db-key" - end - end - end -end diff --git a/test/integration/database_encryption/inspec.yml b/test/integration/database_encryption/inspec.yml deleted file mode 100644 index 4a56322d7e..0000000000 --- a/test/integration/database_encryption/inspec.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: simple_zonal -depends: - - name: inspec-gcp - git: https://github.com/inspec/inspec-gcp.git - tag: v0.10.0 -attributes: - - name: project_id - required: true - type: string - - name: location - required: true - type: string - - name: region - required: true - type: string - - name: cluster_name - required: true - type: string \ No newline at end of file From 353c84c1848b81d3c014e2911fdfc8d2b28be01f Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Tue, 18 Jun 2019 10:30:47 -0700 Subject: [PATCH 14/17] Reverting .kitchen.yml --- .kitchen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kitchen.yml b/.kitchen.yml index e723a711de..7c3e2fea07 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -103,4 +103,4 @@ suites: root_module_directory: test/fixtures/stub_domains_private systems: - name: stub_domains_private - backend: local \ No newline at end of file + backend: local From 055e896c80eabb84d4a39f8616d9b1ad0022f31e Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Tue, 18 Jun 2019 14:10:26 -0700 Subject: [PATCH 15/17] Reverting test/integration/node_pool/controls/kubectl.rb --- test/integration/node_pool/controls/kubectl.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb index e14852f632..8804d6e83e 100644 --- a/test/integration/node_pool/controls/kubectl.rb +++ b/test/integration/node_pool/controls/kubectl.rb @@ -78,12 +78,7 @@ effect: "PreferNoSchedule", key: "all-pools-example", value: "true", - }, - { - effect: "NoSchedule", - key: "nvidia.com/gpu", - value: "present", - }, + } ]) end end From 378aef7e9468b3d15f059e8ea28fc9d286af555c Mon Sep 17 00:00:00 2001 From: Marko Vlahovic Date: Thu, 20 Jun 2019 11:09:02 -0700 Subject: [PATCH 16/17] Moving database_encryption variable under beta module --- README.md | 2 -- autogen/README.md | 2 ++ autogen/cluster_regional.tf | 2 ++ autogen/cluster_zonal.tf | 2 ++ autogen/variables.tf | 30 +++++++++---------- cluster_regional.tf | 3 +- cluster_zonal.tf | 3 +- modules/beta-private-cluster/README.md | 1 + .../beta-private-cluster/cluster_regional.tf | 1 + modules/beta-private-cluster/cluster_zonal.tf | 1 + modules/beta-private-cluster/variables.tf | 17 +++++++++++ modules/beta-public-cluster/README.md | 1 + .../beta-public-cluster/cluster_regional.tf | 1 + modules/beta-public-cluster/cluster_zonal.tf | 1 + modules/beta-public-cluster/variables.tf | 18 ++++++++++- modules/private-cluster/README.md | 2 -- modules/private-cluster/cluster_regional.tf | 1 - modules/private-cluster/cluster_zonal.tf | 1 - modules/private-cluster/variables.tf | 17 ----------- variables.tf | 15 ---------- 20 files changed, 63 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 9c6033bb43..e56d7db123 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o |------|-------------|:----:|:-----:|:-----:| | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| database\_encryption | Application-layer Secrets Encryption settings. Example: database_encryption = [{ state = "ENCRYPTED", key_name = "projects/my-project/locations/global/keyRings/my-ring/cryptoKeys/my-key" }] | list | `` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | @@ -197,7 +196,6 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x -- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 ### Configure a Service Account diff --git a/autogen/README.md b/autogen/README.md index a5a36b4a35..77c45feee6 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -143,7 +143,9 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [Terraform](https://www.terraform.io/downloads.html) 0.11.x {% if private_cluster or beta_cluster %} - [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 +{% else %} - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 +{% endif %} ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index c26191cbcc..bf7d39be3f 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -121,7 +121,9 @@ resource "google_container_cluster" "primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" +{% if beta_cluster %} database_encryption = ["${var.database_encryption}"] +{% endif %} } /****************************************** diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 6c7778a92e..d86342e1cc 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -121,7 +121,9 @@ resource "google_container_cluster" "zonal_primary" { } {% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" +{% if beta_cluster %} database_encryption = ["${var.database_encryption}"] +{% endif %} } /****************************************** diff --git a/autogen/variables.tf b/autogen/variables.tf index 634e49e811..3a7c3e8e5d 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -284,6 +284,21 @@ variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } + +variable "database_encryption" { + description = <` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | diff --git a/modules/beta-private-cluster/cluster_regional.tf b/modules/beta-private-cluster/cluster_regional.tf index 0f93b06e6a..af580bf57e 100644 --- a/modules/beta-private-cluster/cluster_regional.tf +++ b/modules/beta-private-cluster/cluster_regional.tf @@ -115,6 +115,7 @@ resource "google_container_cluster" "primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/beta-private-cluster/cluster_zonal.tf b/modules/beta-private-cluster/cluster_zonal.tf index 72bce00e60..bdac00bd0c 100644 --- a/modules/beta-private-cluster/cluster_zonal.tf +++ b/modules/beta-private-cluster/cluster_zonal.tf @@ -115,6 +115,7 @@ resource "google_container_cluster" "zonal_primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 165a04a735..c435ddf265 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -280,6 +280,23 @@ variable "cloudrun" { default = false } +variable "database_encryption" { + description = <` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | diff --git a/modules/beta-public-cluster/cluster_regional.tf b/modules/beta-public-cluster/cluster_regional.tf index fd0dbe2327..438efff015 100644 --- a/modules/beta-public-cluster/cluster_regional.tf +++ b/modules/beta-public-cluster/cluster_regional.tf @@ -108,6 +108,7 @@ resource "google_container_cluster" "primary" { } } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/beta-public-cluster/cluster_zonal.tf b/modules/beta-public-cluster/cluster_zonal.tf index 734ef9c902..d04c5b4d3e 100644 --- a/modules/beta-public-cluster/cluster_zonal.tf +++ b/modules/beta-public-cluster/cluster_zonal.tf @@ -108,6 +108,7 @@ resource "google_container_cluster" "zonal_primary" { } } remove_default_node_pool = "${var.remove_default_node_pool}" + database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 20cb672223..858b73ae87 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -237,7 +237,6 @@ variable "service_account" { default = "create" } - variable "istio" { description = "(Beta) Enable Istio addon" default = false @@ -248,6 +247,23 @@ variable "cloudrun" { default = false } +variable "database_encryption" { + description = <` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | @@ -210,7 +209,6 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x - [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index beffdc8d4a..5c0818a904 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -108,7 +108,6 @@ resource "google_container_cluster" "primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index b36c6dd66f..39fbcb6098 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -108,7 +108,6 @@ resource "google_container_cluster" "zonal_primary" { master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] } /****************************************** diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index af7592561d..9d6b123ac8 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -284,20 +284,3 @@ variable "issue_client_certificate" { description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" default = "false" } - -variable "database_encryption" { - description = < Date: Thu, 20 Jun 2019 11:16:43 -0700 Subject: [PATCH 17/17] Fixing newline issue in variable.tf --- autogen/variables.tf | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/variables.tf b/autogen/variables.tf index 3a7c3e8e5d..e1d1f50ee1 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -314,4 +314,4 @@ variable "basic_auth_password" { variable "issue_client_certificate" { description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" default = "false" -} \ No newline at end of file +} diff --git a/variables.tf b/variables.tf index b652949bd8..70af17bd71 100644 --- a/variables.tf +++ b/variables.tf @@ -252,4 +252,4 @@ variable "basic_auth_password" { variable "issue_client_certificate" { description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" default = "false" -} \ No newline at end of file +}