diff --git a/autogen/README.md b/autogen/README.md index 778bfdb38e..77c45feee6 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -1,6 +1,6 @@ # Terraform Kubernetes Engine Module -This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc.{% if private_cluster %} This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters){% endif %} +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc.{% if private_cluster %} This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters){% endif %}{% if beta_cluster %}Beta features are enabled on this submodule.{% endif %} The resources/services/activations/deletions that this module will create/trigger are: - Create a GKE cluster with the provided addons @@ -36,6 +36,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" {% endif %} + {% if beta_cluster %} + istio = true + cloudrun = true + {% endif %} node_pools = [ { @@ -137,7 +141,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x -{% if private_cluster %} +{% if private_cluster or beta_cluster %} - [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 {% else %} - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 diff --git a/autogen/auth.tf b/autogen/auth.tf index 3e961cd6b1..a16136fccf 100644 --- a/autogen/auth.tf +++ b/autogen/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" } /****************************************** diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 31b3c600cc..2c5d223014 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -20,7 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -73,6 +73,15 @@ resource "google_container_cluster" "primary" { network_policy_config { disabled = "${var.network_policy ? 0 : 1}" } + {% if beta_cluster %} + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + {% endif %} } ip_allocation_policy { @@ -105,14 +114,12 @@ resource "google_container_cluster" "primary" { } } {% if private_cluster %} - private_cluster_config { enable_private_endpoint = "${var.enable_private_endpoint}" enable_private_nodes = "${var.enable_private_nodes}" master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } {% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" } diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 1c2cab8450..81d371caa4 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -20,7 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -73,6 +73,15 @@ resource "google_container_cluster" "zonal_primary" { network_policy_config { disabled = "${var.network_policy ? 0 : 1}" } + {% if beta_cluster %} + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + {% endif %} } ip_allocation_policy { @@ -105,14 +114,12 @@ resource "google_container_cluster" "zonal_primary" { } } {% if private_cluster %} - private_cluster_config { enable_private_endpoint = "${var.enable_private_endpoint}" enable_private_nodes = "${var.enable_private_nodes}" master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } {% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" } diff --git a/autogen/main.tf b/autogen/main.tf index ad07ed8ac3..ba8bb1c329 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -20,7 +20,7 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" project = "${var.project_id}" region = "${var.region}" } @@ -129,6 +129,20 @@ locals { zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" } + {% if beta_cluster %} + # BETA features + cluster_type_output_istio_enabled = { + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" + } + + cluster_type_output_cloudrun_enabled = { + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.cloudrun_config.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.cloudrun_config.0.disabled, list("")), 0)}" + } + # /BETA features + {% endif %} + cluster_type_output_node_pools_names = { regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" @@ -168,6 +182,12 @@ locals { cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" +{% if beta_cluster %} + # BETA features + cluster_istio_enabled = "${local.cluster_type_output_istio_enabled[local.cluster_type] ? false : true}" + cluster_cloudrun_enabled = "${local.cluster_type_output_cloudrun_enabled[local.cluster_type] ? false : true}" + # /BETA features +{% endif %} {% if private_cluster %} cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? true : false}" {% endif %} diff --git a/autogen/networks.tf b/autogen/networks.tf index 00d41aa4d0..8b1eed188d 100644 --- a/autogen/networks.tf +++ b/autogen/networks.tf @@ -17,13 +17,13 @@ {{ autogeneration_note }} data "google_compute_network" "gke_network" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster %}google-beta{% else %}google{% endif %}" name = "${var.network}" project = "${local.network_project_id}" } data "google_compute_subnetwork" "gke_subnetwork" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "{% if private_cluster %}google-beta{% else %}google{% endif %}" name = "${var.subnetwork}" region = "${var.region}" project = "${local.network_project_id}" diff --git a/autogen/outputs.tf b/autogen/outputs.tf index 191e303bd1..891b49a880 100644 --- a/autogen/outputs.tf +++ b/autogen/outputs.tf @@ -113,10 +113,21 @@ output "service_account" { value = "${local.service_account}" } +{% if beta_cluster %} +output "istio_enabled" { + description = "Whether Istio is enabled" + value = "${local.cluster_istio_enabled}" +} + +output "cloudrun_enabled" { + description = "Whether CloudRun enabled" + value = "${local.cluster_cloudrun_enabled}" +} +{% endif %} + {% if private_cluster %} output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = "${local.cluster_pod_security_policy_enabled}" } - {% endif %} diff --git a/autogen/variables.tf b/autogen/variables.tf index c169f85b65..dc2b86021a 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -251,8 +251,8 @@ variable "service_account" { description = "The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created." default = "create" } -{% if private_cluster %} +{% if private_cluster %} variable "deploy_using_private_endpoint" { description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = "false" @@ -274,6 +274,18 @@ variable "master_ipv4_cidr_block" { } {% endif %} +{% if beta_cluster %} +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} +{% endif %} + variable "basic_auth_username" { description = "The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration." default = "" diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md new file mode 100644 index 0000000000..61779ff9ba --- /dev/null +++ b/examples/simple_regional_beta/README.md @@ -0,0 +1,47 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple cluster with beta features. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf new file mode 100644 index 0000000000..d045ad8bcd --- /dev/null +++ b/examples/simple_regional_beta/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-beta" +} + +provider "google" { + version = "2.3" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "2.3" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +module "gke" { + source = "../../modules/beta-public-cluster/" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = "${var.region}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "${var.compute_engine_service_account}" + istio = "${var.istio}" + cloudrun = "${var.cloudrun}" +} + +data "google_client_config" "default" {} diff --git a/examples/simple_regional_beta/outputs.tf b/examples/simple_regional_beta/outputs.tf new file mode 100644 index 0000000000..9881585a48 --- /dev/null +++ b/examples/simple_regional_beta/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + value = "${module.gke.ca_certificate}" +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_regional_beta/test_outputs.tf b/examples/simple_regional_beta/test_outputs.tf new file mode 100644 index 0000000000..4d1361b7eb --- /dev/null +++ b/examples/simple_regional_beta/test_outputs.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = "${var.project_id}" +} + +output "credentials_path" { + value = "${var.credentials_path}" +} + +output "region" { + value = "${module.gke.region}" +} + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "location" { + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = "${module.gke.zones}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf new file mode 100644 index 0000000000..1da408a790 --- /dev/null +++ b/examples/simple_regional_beta/variables.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to the GCP credentials JSON file" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} diff --git a/examples/simple_regional_private_beta/README.md b/examples/simple_regional_private_beta/README.md new file mode 100644 index 0000000000..e0d005d7b9 --- /dev/null +++ b/examples/simple_regional_private_beta/README.md @@ -0,0 +1,47 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple private cluster with beta features. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf new file mode 100644 index 0000000000..293951501c --- /dev/null +++ b/examples/simple_regional_private_beta/main.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-private-beta" +} + +provider "google-beta" { + version = "2.3" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +data "google_compute_subnetwork" "subnetwork" { + name = "${var.subnetwork}" + project = "${var.project_id}" + region = "${var.region}" +} + +module "gke" { + source = "../../modules/beta-private-cluster/" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = "${var.region}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "${var.compute_engine_service_account}" + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks_config = [{ + cidr_blocks = [{ + cidr_block = "${data.google_compute_subnetwork.subnetwork.ip_cidr_range}" + display_name = "VPC" + }] + }] + + istio = "${var.istio}" + cloudrun = "${var.cloudrun}" +} + +data "google_client_config" "default" {} diff --git a/examples/simple_regional_private_beta/outputs.tf b/examples/simple_regional_private_beta/outputs.tf new file mode 100644 index 0000000000..9881585a48 --- /dev/null +++ b/examples/simple_regional_private_beta/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + value = "${module.gke.ca_certificate}" +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_regional_private_beta/test_outputs.tf b/examples/simple_regional_private_beta/test_outputs.tf new file mode 100644 index 0000000000..4d1361b7eb --- /dev/null +++ b/examples/simple_regional_private_beta/test_outputs.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = "${var.project_id}" +} + +output "credentials_path" { + value = "${var.credentials_path}" +} + +output "region" { + value = "${module.gke.region}" +} + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "location" { + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = "${module.gke.zones}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/simple_regional_private_beta/variables.tf b/examples/simple_regional_private_beta/variables.tf new file mode 100644 index 0000000000..1da408a790 --- /dev/null +++ b/examples/simple_regional_private_beta/variables.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to the GCP credentials JSON file" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index 2a73053a1c..f6beb84832 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -46,7 +46,15 @@ def template_options(self, base): 'private_cluster': False, }), Module("./modules/private-cluster", { + 'private_cluster': True + }), + Module("./modules/beta-private-cluster", { 'private_cluster': True, + 'beta_cluster': True, + }), + Module("./modules/beta-public-cluster", { + 'private_cluster': False, + 'beta_cluster': True, }), ] DEVNULL_FILE = open(os.devnull, 'w') diff --git a/main.tf b/main.tf index e96d301c43..71db7c3f43 100644 --- a/main.tf +++ b/main.tf @@ -161,7 +161,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" - - project = "${var.project_id}" + zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + project = "${var.project_id}" } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md new file mode 100644 index 0000000000..b1d3d75b91 --- /dev/null +++ b/modules/beta-private-cluster/README.md @@ -0,0 +1,283 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters)Beta features are enabled on this submodule. +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `network_policy` is true + +## Usage +There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: + +```hcl +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + http_load_balancing = false + horizontal_pod_autoscaling = true + kubernetes_dashboard = true + network_policy = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "10.0.0.0/28" + istio = true + cloudrun = true + + node_pools = [ + { + name = "default-node-pool" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 100 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + }, + ] + + node_pools_labels = { + all = {} + + default-node-pool = { + default-node-pool = "true" + } + } + + node_pools_metadata = { + all = {} + + default-node-pool = { + node-pool-metadata-custom-value = "my-node-pool" + } + } + + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = "true" + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + + node_pools_tags = { + all = [] + + default-node-pool = [ + "default-node-pool", + ] + } +} +``` + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + +## Upgrade to v1.0.0 + +Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. + +In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. + +[^]: (autogen_docs_start) +[^]: (autogen_docs_end) + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.0.0 + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountAdmin +- roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +## File structure +The project has the following folders and files: + +- /: root folder +- /examples: examples for using this module +- /helpers: Helper scripts +- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file) +- /test: Folders with files for testing the module (see Testing section on this file) +- /main.tf: main file for this module, contains all the resources to create +- /variables.tf: all the variables for the module +- /output.tf: the outputs of the module +- /readme.MD: this file + +## Templating + +To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](/autogen) directory. + +The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate. + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + +Six test-kitchen instances are defined: + +- `deploy-service` +- `node-pool` +- `shared-vpc` +- `simple-regional` +- `simple-zonal` +- `stub-domains` + +The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory. + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. + - Requires the [permissions to run the module](#configure-a-service-account) + - Requires `roles/compute.networkAdmin` to create the test suite's networks + - Requires `roles/resourcemanager.projectIamAdmin` since service account creation is tested +3. Build the Docker container for testing: + + ``` + make docker_build_kitchen_terraform + ``` +4. Run the testing container in interactive mode: + + ``` + make docker_run + ``` + + The module root directory will be loaded into the Docker container at `/cft/workdir/`. +5. Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. Run `kitchen converge` again. This is necessary due to an oddity in how `networkPolicyConfig` is handled by the upstream API. (See [#72](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/72) for details). + 4. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 5. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. + +If you wish to parallelize running the test suites, it is also possible to offload the work onto Concourse to run each test suite for you using the command `make test_integration_concourse`. The `.concourse` directory will be created and contain all of the logs from the running test suites. + +When running tests locally, you will need to use your own test project environment. You can configure your environment by setting all of the following variables: + +``` +export COMPUTE_ENGINE_SERVICE_ACCOUNT="" +export PROJECT_ID="" +export REGION="" +export ZONES='[""]' +export SERVICE_ACCOUNT_JSON="$(cat "")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" +export GOOGLE_APPLICATION_CREDENTIALS="" +``` + +#### Test configuration + +Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. +For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. +Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. + +Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running hadolint on Dockerfiles +Checking for required files +Testing the validity of the header check +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.026s + +OK +Checking file headers +The following lines have trailing whitespace +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Python - flake8. Can be installed with 'pip install flake8' +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/modules/beta-private-cluster/auth.tf b/modules/beta-private-cluster/auth.tf new file mode 100644 index 0000000000..56e015b1ef --- /dev/null +++ b/modules/beta-private-cluster/auth.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Retrieve authentication token + *****************************************/ +data "google_client_config" "default" { + provider = "google-beta" +} + +/****************************************** + Configure provider + *****************************************/ +provider "kubernetes" { + load_config_file = false + host = "https://${local.cluster_endpoint}" + token = "${data.google_client_config.default.access_token}" + cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" +} \ No newline at end of file diff --git a/modules/beta-private-cluster/cluster_regional.tf b/modules/beta-private-cluster/cluster_regional.tf new file mode 100644 index 0000000000..ad81c86f37 --- /dev/null +++ b/modules/beta-private-cluster/cluster_regional.tf @@ -0,0 +1,169 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create regional cluster + *****************************************/ +resource "google_container_cluster" "primary" { + provider = "google-beta" + count = "${var.regional ? 1 : 0}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + region = "${var.region}" + additional_zones = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version_regional}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + } + } + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create regional node pools + *****************************************/ +resource "google_container_node_pool" "pools" { + provider = "google-beta" + count = "${var.regional ? length(var.node_pools) : 0}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + region = "${var.region}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.primary"] +} + +resource "null_resource" "wait_for_regional_cluster" { + count = "${var.regional ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] +} \ No newline at end of file diff --git a/modules/beta-private-cluster/cluster_zonal.tf b/modules/beta-private-cluster/cluster_zonal.tf new file mode 100644 index 0000000000..62e8435d78 --- /dev/null +++ b/modules/beta-private-cluster/cluster_zonal.tf @@ -0,0 +1,169 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create zonal cluster + *****************************************/ +resource "google_container_cluster" "zonal_primary" { + provider = "google-beta" + count = "${var.regional ? 0 : 1}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + zone = "${var.zones[0]}" + additional_zones = ["${slice(var.zones,1,length(var.zones))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version_zonal}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + } + } + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create zonal node pools + *****************************************/ +resource "google_container_node_pool" "zonal_pools" { + provider = "google-beta" + count = "${var.regional ? 0 : length(var.node_pools)}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + zone = "${var.zones[0]}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.zonal_primary"] +} + +resource "null_resource" "wait_for_zonal_cluster" { + count = "${var.regional ? 0 : 1}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} \ No newline at end of file diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf new file mode 100644 index 0000000000..56d702e00a --- /dev/null +++ b/modules/beta-private-cluster/dns.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Delete default kube-dns configmap + *****************************************/ +resource "null_resource" "delete_default_kube_dns_configmap" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" + } + + depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} + +/****************************************** + Create kube-dns confimap + *****************************************/ +resource "kubernetes_config_map" "kube-dns" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + metadata { + name = "kube-dns" + namespace = "kube-system" + + labels { + maintained_by = "terraform" + } + } + + data { + stubDomains = <&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" -o json "${RESOURCE_NAME}" | jq -r '.metadata.labels."maintained_by"') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi \ No newline at end of file diff --git a/modules/beta-private-cluster/scripts/kubectl_wrapper.sh b/modules/beta-private-cluster/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..255c128223 --- /dev/null +++ b/modules/beta-private-cluster/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" \ No newline at end of file diff --git a/modules/beta-private-cluster/scripts/wait-for-cluster.sh b/modules/beta-private-cluster/scripts/wait-for-cluster.sh new file mode 100755 index 0000000000..3ea1dd78ed --- /dev/null +++ b/modules/beta-private-cluster/scripts/wait-for-cluster.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [[ "${current_status}" == "RECONCILING" ]]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" \ No newline at end of file diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf new file mode 100644 index 0000000000..aa281d5d4c --- /dev/null +++ b/modules/beta-private-cluster/variables.tf @@ -0,0 +1,243 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + description = "The name of the cluster (required)" +} + +variable "description" { + description = "The description of the cluster" + default = "" +} + +variable "regional" { + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = "list" + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [""] +} + +variable "network" { + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} + +variable "node_version" { + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = "list" + + description = <` to create resources for a specific test case. + 3. Run `kitchen converge` again. This is necessary due to an oddity in how `networkPolicyConfig` is handled by the upstream API. (See [#72](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/72) for details). + 4. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 5. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. + +If you wish to parallelize running the test suites, it is also possible to offload the work onto Concourse to run each test suite for you using the command `make test_integration_concourse`. The `.concourse` directory will be created and contain all of the logs from the running test suites. + +When running tests locally, you will need to use your own test project environment. You can configure your environment by setting all of the following variables: + +``` +export COMPUTE_ENGINE_SERVICE_ACCOUNT="" +export PROJECT_ID="" +export REGION="" +export ZONES='[""]' +export SERVICE_ACCOUNT_JSON="$(cat "")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" +export GOOGLE_APPLICATION_CREDENTIALS="" +``` + +#### Test configuration + +Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. +For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. +Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. + +Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running hadolint on Dockerfiles +Checking for required files +Testing the validity of the header check +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.026s + +OK +Checking file headers +The following lines have trailing whitespace +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Python - flake8. Can be installed with 'pip install flake8' +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/modules/beta-public-cluster/auth.tf b/modules/beta-public-cluster/auth.tf new file mode 100644 index 0000000000..56e015b1ef --- /dev/null +++ b/modules/beta-public-cluster/auth.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Retrieve authentication token + *****************************************/ +data "google_client_config" "default" { + provider = "google-beta" +} + +/****************************************** + Configure provider + *****************************************/ +provider "kubernetes" { + load_config_file = false + host = "https://${local.cluster_endpoint}" + token = "${data.google_client_config.default.access_token}" + cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" +} \ No newline at end of file diff --git a/modules/beta-public-cluster/cluster_regional.tf b/modules/beta-public-cluster/cluster_regional.tf new file mode 100644 index 0000000000..9a8714a723 --- /dev/null +++ b/modules/beta-public-cluster/cluster_regional.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create regional cluster + *****************************************/ +resource "google_container_cluster" "primary" { + provider = "google-beta" + count = "${var.regional ? 1 : 0}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + region = "${var.region}" + additional_zones = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version_regional}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + } + } + + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create regional node pools + *****************************************/ +resource "google_container_node_pool" "pools" { + provider = "google-beta" + count = "${var.regional ? length(var.node_pools) : 0}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + region = "${var.region}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.primary"] +} + +resource "null_resource" "wait_for_regional_cluster" { + count = "${var.regional ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] +} diff --git a/modules/beta-public-cluster/cluster_zonal.tf b/modules/beta-public-cluster/cluster_zonal.tf new file mode 100644 index 0000000000..3f3ab40a61 --- /dev/null +++ b/modules/beta-public-cluster/cluster_zonal.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create zonal cluster + *****************************************/ +resource "google_container_cluster" "zonal_primary" { + provider = "google-beta" + count = "${var.regional ? 0 : 1}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + zone = "${var.zones[0]}" + additional_zones = ["${slice(var.zones,1,length(var.zones))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version_zonal}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + istio_config { + disabled = "${var.istio ? 0 : 1}" + } + + cloudrun_config { + disabled = "${var.cloudrun ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + } + } + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create zonal node pools + *****************************************/ +resource "google_container_node_pool" "zonal_pools" { + provider = "google-beta" + count = "${var.regional ? 0 : length(var.node_pools)}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + zone = "${var.zones[0]}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.zonal_primary"] +} + +resource "null_resource" "wait_for_zonal_cluster" { + count = "${var.regional ? 0 : 1}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} \ No newline at end of file diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf new file mode 100644 index 0000000000..56d702e00a --- /dev/null +++ b/modules/beta-public-cluster/dns.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Delete default kube-dns configmap + *****************************************/ +resource "null_resource" "delete_default_kube_dns_configmap" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" + } + + depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} + +/****************************************** + Create kube-dns confimap + *****************************************/ +resource "kubernetes_config_map" "kube-dns" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + metadata { + name = "kube-dns" + namespace = "kube-system" + + labels { + maintained_by = "terraform" + } + } + + data { + stubDomains = <&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" -o json "${RESOURCE_NAME}" | jq -r '.metadata.labels."maintained_by"') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi \ No newline at end of file diff --git a/modules/beta-public-cluster/scripts/kubectl_wrapper.sh b/modules/beta-public-cluster/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..255c128223 --- /dev/null +++ b/modules/beta-public-cluster/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" \ No newline at end of file diff --git a/modules/beta-public-cluster/scripts/wait-for-cluster.sh b/modules/beta-public-cluster/scripts/wait-for-cluster.sh new file mode 100755 index 0000000000..3ea1dd78ed --- /dev/null +++ b/modules/beta-public-cluster/scripts/wait-for-cluster.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [[ "${current_status}" == "RECONCILING" ]]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" \ No newline at end of file diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf new file mode 100644 index 0000000000..4ffb499a60 --- /dev/null +++ b/modules/beta-public-cluster/variables.tf @@ -0,0 +1,229 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + description = "The name of the cluster (required)" +} + +variable "description" { + description = "The description of the cluster" + default = "" +} + +variable "regional" { + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = "list" + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [""] +} + +variable "network" { + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} + +variable "node_version" { + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = "list" + + description = <