From 54c58f67e60fd834cbd42c91d6347711a0be1def Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 19 Apr 2021 21:33:22 -0400 Subject: [PATCH] infra/gcp/main: create gcs buckets for terraform Allow groups less privileged than k8s-infra-gcp-org-admins to use terraform to manage resources over which they have ownership. Terraform state can include potentially include sensitive values. Since we have terraform setup to store state in GCS, we need to ensure visibility and access to state matches ownership of (privileges to modify) the resources it describes. We're using uniform bucket-level access on our GCS buckets to avoid the complexity introduced by per-object ACLs. This means if we want different groups with different privilege levels using terraform to manage different sets of resources, we need to provision a GCS bucket for each group. The new bucket schema is "k8s-infra-tf-{folder}[-{suffix}]" where: - {folder} is the intended GCP folder for GCP projects managed by this group, access level should be ~owners of folder - {suffix} is subset of resources contained somewhere underneath folder, access level should ~editors of those resources The GCP folders don't actually exist yet, but the plan is: - public: kubernetes-public (potentially release related projects too) - prow: prow-build clusters and e2e projects - aws: if there are gcp projects being used to manage aws resources - sandbox: temporary projects The buckets being added are: - k8s-infra-tf-aws: to manage aws resources - k8s-infra-tf-prow-clusters: to manage prow-build, prow-build-trusted - k8s-infra-tf-public-clusters: to manage aaa - k8s-infra-tf-sandbox-ii: for the ii team to manage things in sandbox Organization admins are given storage.admin privileges to all buckets for break-glass purposes. Terraform modules were migrated by running `terraform init` and `terraform refresh` for each module --- .../projects/k8s-infra-ii-sandbox/provider.tf | 4 +- .../prow-build-trusted/00-provider.tf | 2 +- .../prow-build/00-provider.tf | 2 +- .../kubernetes-public/aaa/00-inputs.tf | 10 +++-- infra/gcp/ensure-main-project.sh | 44 +++++++++++++------ 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/infra/gcp/clusters/projects/k8s-infra-ii-sandbox/provider.tf b/infra/gcp/clusters/projects/k8s-infra-ii-sandbox/provider.tf index 2972de2b389..dc2e4e64975 100644 --- a/infra/gcp/clusters/projects/k8s-infra-ii-sandbox/provider.tf +++ b/infra/gcp/clusters/projects/k8s-infra-ii-sandbox/provider.tf @@ -7,7 +7,9 @@ This file defines: terraform { backend "gcs" { - bucket = "k8s-infra-clusters-terraform" + bucket = "k8s-infra-tf-sandbox-ii" + // TODO(spiffxp): the names not matching weirds me out a bit, it would be + // nice to rename the project at some point prefix = "k8s-infra-ii-sandbox" } diff --git a/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/00-provider.tf b/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/00-provider.tf index 5cfc09c2808..cbf797d802b 100644 --- a/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/00-provider.tf +++ b/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/00-provider.tf @@ -7,7 +7,7 @@ This file defines: terraform { backend "gcs" { - bucket = "k8s-infra-clusters-terraform" + bucket = "k8s-infra-tf-prow-clusters" prefix = "k8s-infra-prow-build-trusted/prow-build-trusted" // $project_name/$cluster_name } diff --git a/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/00-provider.tf b/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/00-provider.tf index 532298b8020..f76ce541dc3 100644 --- a/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/00-provider.tf +++ b/infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/00-provider.tf @@ -7,7 +7,7 @@ This file defines: terraform { backend "gcs" { - bucket = "k8s-infra-clusters-terraform" + bucket = "k8s-infra-tf-prow-clusters" prefix = "k8s-infra-prow-build/prow-build" // $project_name/$cluster_name } diff --git a/infra/gcp/clusters/projects/kubernetes-public/aaa/00-inputs.tf b/infra/gcp/clusters/projects/kubernetes-public/aaa/00-inputs.tf index 4a6ca070a50..13e599a88b1 100644 --- a/infra/gcp/clusters/projects/kubernetes-public/aaa/00-inputs.tf +++ b/infra/gcp/clusters/projects/kubernetes-public/aaa/00-inputs.tf @@ -10,13 +10,17 @@ terraform { required_version = ">= 0.12.8" backend "gcs" { - bucket = "k8s-infra-clusters-terraform" + bucket = "k8s-infra-tf-public-clusters" prefix = "kubernetes-public/aaa" // $project_name/$cluster_name } required_providers { - google = "~> 2.14" - google-beta = "~> 2.14" + google = { + version = "~> 2.14" + } + google-beta = { + version = "~> 2.14" + } } } diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh index d797d526cf6..9859cabe311 100755 --- a/infra/gcp/ensure-main-project.sh +++ b/infra/gcp/ensure-main-project.sh @@ -49,7 +49,7 @@ CLUSTER_ADMINS_GROUP="k8s-infra-cluster-admins@kubernetes.io" ACCOUNTING_GROUP="k8s-infra-gcp-accounting@kubernetes.io" # The GCS bucket which hold terraform state for clusters -CLUSTER_TERRAFORM_BUCKET="k8s-infra-clusters-terraform" +LEGACY_CLUSTER_TERRAFORM_BUCKET="k8s-infra-clusters-terraform" # The GKE security groups group CLUSTER_USERS_GROUP="gke-security-groups@kubernetes.io" @@ -74,10 +74,35 @@ apis=( ) ensure_only_services "${PROJECT}" "${apis[@]}" -color 6 "Ensuring the cluster terraform-state bucket exists" -ensure_private_gcs_bucket \ - "${PROJECT}" \ - "gs://${CLUSTER_TERRAFORM_BUCKET}" +# buckets to hold terraform state +# - since we are using uniform bucket level access (ubla), each bucket should +# represent a logical group of access, with org admins given storage.admin +# for break-glass purposes +# - the legacy bucket (k8s-infra-clusters-terraform) assumed the same set of +# users should have access to all gke clusters +# - new bucket schema is "k8s-infra-tf-{folder}[-{suffix}]" where: +# - folder: intended GCP folder for GCP projects managed by this terraform, +# access level is ~owners of folder +# - suffix: some subset of resources contained somewhere underneath folder, +# access level is ~editors of those resources +# - entry syntax is "bucket_name:owners_group" (: is invalid bucket name char) +terraform_state_bucket_entries=( + "${LEGACY_CLUSTER_TERRAFORM_BUCKET}:${CLUSTER_ADMINS_GROUP}" + k8s-infra-tf-aws:k8s-infra-aws-admins@kubernetes.io + k8s-infra-tf-prow-clusters:k8s-infra-prow-oncall@kubernetes.io + k8s-infra-tf-public-clusters:"${CLUSTER_ADMINS_GROUP}" + k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io +) +color 6 "Ensuring terraform state buckets exist with correct permissions" +for entry in "${terraform_state_bucket_entries[@]}"; do + bucket="gs://$(echo "${entry}" | cut -d: -f1)" + owners="$(echo "${entry}" | cut -d: -f2-)" + color 6 "Ensuring '${bucket}' exists as private with owners '${owners}'"; ( + ensure_private_gcs_bucket "${PROJECT}" "${bucket}" + empower_group_to_admin_gcs_bucket "${owners}" "${bucket}" + ensure_gcs_role_binding "${bucket}" "group:k8s-infra-gcp-org-admins@kubernetes.io" "roles/storage.admin" + ) 2>&1 | indent +done 2>&1 | indent color 6 "Empowering BigQuery admins" ensure_project_role_binding \ @@ -100,15 +125,6 @@ done ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)" ensure_removed_custom_project_iam_role "${PROJECT}" "ServiceAccountLister" -color 6 "Empowering cluster admins to own gs://${CLUSTER_TERRAFORM_BUCKET}" -ensure_gcs_role_binding \ - "gs://${CLUSTER_TERRAFORM_BUCKET}" \ - "group:${CLUSTER_ADMINS_GROUP}" \ - "objectAdmin" -ensure_gcs_role_binding \ - "gs://${CLUSTER_TERRAFORM_BUCKET}" \ - "group:${CLUSTER_ADMINS_GROUP}" \ - "legacyBucketOwner" color 6 "Empowering cluster users" ensure_project_role_binding \