Merge pull request #2940 from spiffxp/k8s-infra-org-tf

terraform/k8s-infra-kubernetes-io: manage k8s-infra-sandbox-capg budget

infra/gcp/bash/ensure-main-project.sh

@@ -74,12 +74,13 @@ readonly DNS_GROUP="k8s-infra-dns-admins@kubernetes.io"
 readonly TERRAFORM_STATE_BUCKET_ENTRIES=(
     "${LEGACY_CLUSTER_TERRAFORM_BUCKET}:${CLUSTER_ADMINS_GROUP}"
     k8s-infra-tf-aws:k8s-infra-aws-admins@kubernetes.io
+    k8s-infra-tf-gcp:k8s-infra-gcp-org-admins@kubernetes.io
     k8s-infra-tf-monitoring:"${CLUSTER_ADMINS_GROUP}"
     k8s-infra-tf-prow-clusters:k8s-infra-prow-oncall@kubernetes.io
     k8s-infra-tf-public-clusters:"${CLUSTER_ADMINS_GROUP}"
     k8s-infra-tf-public-pii:"${CLUSTER_ADMINS_GROUP}"
-    k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io
     k8s-infra-tf-sandbox-capg:k8s-infra-sandbox-capg@kubernetes.io
+    k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io
 )
 
 # The services we explicitly want enabled for the main project

infra/gcp/terraform/k8s-infra-kubernetes-io/main.tf

@@ -0,0 +1,139 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+data "google_billing_account" "account" {
+  billing_account = "018801-93540E-22A20E"
+}
+
+data "google_organization" "org" {
+  domain = "kubernetes.io"
+}
+
+resource "google_project" "project" {
+  name            = "k8s-infra-kubernetes-io"
+  project_id      = "k8s-infra-kubernetes-io"
+  org_id          = data.google_organization.org.org_id
+  billing_account = data.google_billing_account.account.billing_account
+}
+
+resource "google_project_service" "project" {
+  project = google_project.project.id
+  for_each = toset([
+    "billingbudgets.googleapis.com",
+    "cloudbilling.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
+    "monitoring.googleapis.com",
+    "serviceusage.googleapis.com",
+  ])
+  service = each.value
+}
+
+data "google_project" "kubernetes_public" {
+  project_id = "kubernetes-public"
+}
+
+resource "google_resource_manager_lien" "kubernetes_public" {
+  parent = "projects/${data.google_project.kubernetes_public.number}"
+  restrictions = ["resourcemanager.projects.delete"]
+  origin = "do-not-delete-kubernetes-public"
+  reason = "kubernetes-public hosts public-facing kubernetes project infrastructure"
+}
+
+data "google_project" "k8s_infra_sandbox_capg" {
+  project_id = "k8s-infra-sandbox-capg"
+}
+
+data "google_monitoring_notification_channel" "sig_k8s_infra_leads" {
+  project = data.google_project.kubernetes_public.project_id
+  display_name = "sig-k8s-infra-leads@kubernetes.io"
+}
+
+resource "google_billing_budget" "capg_budget" {
+  billing_account = data.google_billing_account.account.billing_account
+  display_name = "k8s-infra-sandbox-capg"
+  budget_filter {
+    # calendar_period = "MONTH" # TODO: terraform doesn't support this?
+    projects = [ "projects/${data.google_project.k8s_infra_sandbox_capg.number}" ]
+    # exclude promotions, which is where our credits come from, since that zeros out cost
+    credit_types_treatment = "INCLUDE_SPECIFIED_CREDITS"
+    credit_types           = [
+        "SUSTAINED_USAGE_DISCOUNT",
+        "DISCOUNT",
+        "COMMITTED_USAGE_DISCOUNT",
+        "FREE_TIER",
+        "COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE",
+        "SUBSCRIPTION_BENEFIT",
+    ]
+  }
+  amount {
+    specified_amount {
+      currency_code = "USD"
+      units = "5000"
+    }
+  }
+  all_updates_rule {
+    # Don't send to users with Billing Account Administrators and
+    # Billing Account Users IAM roles for the billing account
+    disable_default_iam_recipients = true
+    monitoring_notification_channels = [
+      data.google_monitoring_notification_channel.sig_k8s_infra_leads.name
+    ]
+  }
+  dynamic "threshold_rules" {
+    for_each = toset([0.2, 0.5, 0.9, 1.0])
+    content {
+      threshold_percent = threshold_rules.value
+    }
+  }
+}
+
+resource "google_billing_budget" "k8s_infra" {
+  billing_account = data.google_billing_account.account.billing_account
+  display_name = "k8s-infra-monthly"
+  budget_filter {
+    # calendar_period = "MONTH" # TODO: terraform doesn't support this?
+    # exclude promotions, which is where our credits come from, since that zeros out cost
+    credit_types_treatment = "INCLUDE_SPECIFIED_CREDITS"
+    credit_types           = [
+        "SUSTAINED_USAGE_DISCOUNT",
+        "DISCOUNT",
+        "COMMITTED_USAGE_DISCOUNT",
+        "FREE_TIER",
+        "COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE",
+        "SUBSCRIPTION_BENEFIT",
+    ]
+  }
+  amount {
+    specified_amount {
+      currency_code = "USD"
+      units = "250000" # 3M/yr / 12mo
+    }
+  }
+  all_updates_rule {
+    # Don't send to users with Billing Account Administrators and
+    # Billing Account Users IAM roles for the billing account
+    disable_default_iam_recipients = true
+    monitoring_notification_channels = [
+      data.google_monitoring_notification_channel.sig_k8s_infra_leads.name
+    ]
+  }
+  dynamic "threshold_rules" {
+    for_each = toset([0.9, 1.0])
+    content {
+      threshold_percent = threshold_rules.value
+    }
+  }
+}

infra/gcp/terraform/k8s-infra-kubernetes-io/provider.tf

@@ -0,0 +1,50 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This file defines:
+- Required provider versions
+- Storage backend details
+*/
+
+terraform {
+
+  backend "gcs" {
+    bucket = "k8s-infra-tf-gcp"
+    prefix = "kubernetes-io"
+  }
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 3.87.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 3.87.0"
+    }
+  }
+}
+
+provider "google" {
+  # necessary to create google_billing_budget resources
+  # NOTE: bootstrapping was necessary to use this, involved setting to false
+  #       and creating the project / enabling the required services using
+  #       another gcloud project with cloudresourcemanager and serviceusage
+  #       APIs enabled
+  user_project_override = true
+  billing_project = "k8s-infra-kubernetes-io"
+}

infra/gcp/terraform/k8s-infra-kubernetes-io/versions.tf

@@ -0,0 +1,24 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This file defines:
+- Required Terraform version
+*/
+
+terraform {
+  required_version = "~> 1.0.0"
+}