From 8b36235558ff75b2096b12cdd0329f0db8205004 Mon Sep 17 00:00:00 2001
From: Alex Chen <alexyfchen@google.com>
Date: Tue, 28 Feb 2023 20:12:46 -0800
Subject: [PATCH 1/4] [Custom DC] Various setup improvements

---
 .../dc_website/templates/deployment.yaml      |  4 +-
 .../dc_website/templates/ingress.yaml         |  5 +-
 deploy/helm_charts/dc_website/values.yaml     | 14 ++--
 .../examples/website_v1/.terraform.lock.hcl   | 79 -------------------
 .../examples/website_v1/main.tf               | 49 +-----------
 .../examples/website_v1/outputs.tf            |  4 +-
 .../examples/website_v1/variables.tf          | 17 +++-
 .../modules/apikeys/main.tf                   |  3 +-
 .../modules/gke/configure_cluster.sh          |  7 +-
 .../modules/gke/create_cluster.sh             | 24 ------
 .../modules/gke/main.tf                       | 69 ++++++++++++----
 .../modules/gke/outputs.tf                    |  4 +-
 .../modules/gke/variables.tf                  |  6 +-
 scripts/install_custom_dc.sh                  | 50 +++++++++++-
 14 files changed, 144 insertions(+), 191 deletions(-)
 delete mode 100644 deploy/terraform-datacommons-website/examples/website_v1/.terraform.lock.hcl
 delete mode 100644 deploy/terraform-datacommons-website/modules/gke/create_cluster.sh

diff --git a/deploy/helm_charts/dc_website/templates/deployment.yaml b/deploy/helm_charts/dc_website/templates/deployment.yaml
index 008e5292fa..a94d76b337 100644
--- a/deploy/helm_charts/dc_website/templates/deployment.yaml
+++ b/deploy/helm_charts/dc_website/templates/deployment.yaml
@@ -56,7 +56,7 @@ spec:
             name: schema-mapping{{ .Values.resourceSuffix }}
       containers:
         - name: website
-          image:  "{{ .Values.website.image.repository }}:{{ .Values.website.image.tag }}"
+          image:  "{{ .Values.website.image.registry }}/{{ .Values.website.image.project }}/{{ .Values.website.image.repository }}:{{ .Values.website.image.tag }}"
           imagePullPolicy: {{ .Values.website.image.pullPolicy }}
           args: []
           ports:
@@ -117,7 +117,7 @@ spec:
             - name: BOUNCE
               value: "dummy"
         - name: mixer
-          image:  "{{ .Values.mixer.image.repository }}:{{ .Values.mixer.image.tag | default .Chart.AppVersion }}"
+          image:  "{{ .Values.mixer.image.registry }}/{{ .Values.mixer.image.project }}/{{ .Values.mixer.image.repository }}:{{ .Values.mixer.image.tag }}"
           imagePullPolicy: {{ .Values.mixer.image.pullPolicy }}
           resources:
             limits:
diff --git a/deploy/helm_charts/dc_website/templates/ingress.yaml b/deploy/helm_charts/dc_website/templates/ingress.yaml
index a979654c6a..bcf0c9f3b5 100644
--- a/deploy/helm_charts/dc_website/templates/ingress.yaml
+++ b/deploy/helm_charts/dc_website/templates/ingress.yaml
@@ -29,10 +29,9 @@ kind: Ingress
 metadata:
   name: {{ .Values.ingress.name }}
   namespace: {{ .Values.namespace.name }}
-  {{- with .Values.ingress.annotations }}
   annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
+    kubernetes.io/ingress.global-static-ip-name: dc-website-ip
+    ingress.gcp.kubernetes.io/pre-shared-cert: dc-website-cert{{ .Values.resourceSuffix }}
 spec:
   rules:
     - http:
diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml
index c341e43a46..c1e0d148d5 100644
--- a/deploy/helm_charts/dc_website/values.yaml
+++ b/deploy/helm_charts/dc_website/values.yaml
@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-resourceSuffix:
+resourceSuffix: ""
 
 # Website service config.
 website:
@@ -13,11 +13,13 @@ website:
   githash:
 
   image:
-    repository: gcr.io/datcom-ci/datacommons-website
+    registry: "gcr.io"
+    project: "datcom-ci"
+    repository: "datacommons-website"
     pullPolicy: Always
     tag:
 
-  flaskEnv:
+  flaskEnv: "custom"
   secretGCPProjectID:
   enableModel: false
 
@@ -45,7 +47,6 @@ serviceAccount:
 ingress:
   name: website-ingress
   enabled:
-  annotations: { kubernetes.io/ingress.global-static-ip-name: mixer-ip }
 
 ###############################################################################
 # Config for Mixer helm chart
@@ -55,7 +56,10 @@ mixer:
   githash:
 
   image:
-    repository: gcr.io/datcom-ci/datacommons-mixer
+    registry: "gcr.io"
+    project: "datcom-ci"
+    repository: "datacommons-mixer"
+
     pullPolicy: Always
     tag:
 
diff --git a/deploy/terraform-datacommons-website/examples/website_v1/.terraform.lock.hcl b/deploy/terraform-datacommons-website/examples/website_v1/.terraform.lock.hcl
deleted file mode 100644
index 0caaddcea0..0000000000
--- a/deploy/terraform-datacommons-website/examples/website_v1/.terraform.lock.hcl
+++ /dev/null
@@ -1,79 +0,0 @@
-# This file is maintained automatically by "terraform init".
-# Manual edits may be lost in future updates.
-
-provider "registry.terraform.io/hashicorp/external" {
-  version     = "2.0.0"
-  constraints = "~> 2.0.0"
-  hashes = [
-    "h1:6S7hqjmUnoAZ5D/0F1VlJZKSJsUIBh7Ro0tLjGpKO0g=",
-    "zh:07949780dd6a1d43e7b46950f6e6976581d9724102cb5388d3411a1b6f476bde",
-    "zh:0a4f4636ff93f0644affa8474465dd8c9252946437ad025b28fc9f6603534a24",
-    "zh:0dd7e05a974c649950d1a21d7015d3753324ae52ebdd1744b144bc409ca4b3e8",
-    "zh:2b881032b9aa9d227ac712f614056d050bcdcc67df0dc79e2b2cb76a197059ad",
-    "zh:38feb4787b4570335459ca75a55389df1a7570bdca8cdf5df4c2876afe3c14b4",
-    "zh:40f7e0aaef3b1f4c2ca2bb1189e3fe9af8c296da129423986d1d99ccc8cfb86c",
-    "zh:56b361f64f0f0df5c4f958ae2f0e6f8ba192f35b720b9d3ae1be068fabcf73d9",
-    "zh:5fadb5880cd31c2105f635ded92b9b16f918c1dd989627a4ce62c04939223909",
-    "zh:61fa0be9c14c8c4109cfb7be8d54a80c56d35dbae49d3231cddb59831e7e5a4d",
-    "zh:853774bf97fbc4a784d5af5a4ca0090848430781ae6cfc586adeb48f7c44af79",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/google" {
-  version     = "4.28.0"
-  constraints = ">= 3.43.0, ~> 4.28.0, < 5.0.0"
-  hashes = [
-    "h1:GWIsjFFxrWk2kY+xrzfczjCCBL2m2WuI5/Kw3AF5y2Q=",
-    "zh:17664192fbeb733d6d6cfa17fbd1c54e6f1614f635f48adfae17557e121b63eb",
-    "zh:2993a3ba417c576ca9c6adccb6a6e914b4dedd3f91a735fd14ab8910936d8c11",
-    "zh:452323359fd64dc0fbf96da8c1df1df57cacef72cf2615631662bdec1d73d94e",
-    "zh:492c1d1bfbd9bef2a30fab0096ae642ceeeee81499cf5aa9f4505a884b0855a3",
-    "zh:611875b0246bdbd8815f8e81e744e31466559fec3b4566c9d2f3d1fd54c20292",
-    "zh:63c5084e1ac50165da1feebf2f51af3c8a7b61f817861418850b2b59b010b604",
-    "zh:6efb784223405839aa22fc6e40e37e08dd7ba37310e327dea1731299e5c67104",
-    "zh:ac51b5555bfdee282885475831bdc336f42294687e887f91cd339f15a4f69bc1",
-    "zh:c98e971f99f43aea9e0363cdd478e3b19f79b4553357a089bb10ba7ab897a932",
-    "zh:e9c29205674657f7b31f352a680a17262a797150bcbe76b26939d5cb39d19199",
-    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:f968998e7690cd508c1f7847d7fa3ba2ae448d70e94414ada85c1cee81b3bcd9",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "4.28.0"
-  constraints = ">= 3.43.0, ~> 4.28.0, < 5.0.0"
-  hashes = [
-    "h1:OCnAHwByjG7Ck54UXweixWOKuQfheAgIbLvs3Rhttws=",
-    "zh:02049634b3dd7928628145c1993e9a6772b8229e94c0466943dd8a192c7dcf43",
-    "zh:113664260f56d0559c9f4c5b912dd80ee966d09ff3723fe6ae1a71fa4915fdc9",
-    "zh:28de139a2db9ccf280a92fa18ed41f36ea5ef4269fda4124288751eec45b6907",
-    "zh:303dec5be87bd935351bf991da4edff71bab9909ce410a819b3dd0849a27df8b",
-    "zh:579b6cf837488a0e6335c1ca0b81ce0936d2ea29c24b4fb8ba54018e81a0cabb",
-    "zh:77cdb315e9144739241f9ea3e55502104dece33ce0acd9694469a3a5df4e3906",
-    "zh:837c37d168dc557b474b5dac3b850e134779a27ee9df3f49a4427c569d0eae44",
-    "zh:9359bf058b95fa6b9337a3b55168517fd380e6752c383c964fb776513621aca4",
-    "zh:cf3cdef5ed5d4a321ffd2cac070a00ff0f8cee7bfd6a2697c494a1d06937bb67",
-    "zh:e4f647bd336260fc477f7ab77e48e825d49f3d4ed1391bf232b5039cfc411760",
-    "zh:ec3a02205594beeeedd090dd6c831a988fc0ff58fb353cc78dc6395eedf19979",
-    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/null" {
-  version = "3.1.1"
-  hashes = [
-    "h1:Pctug/s/2Hg5FJqjYcTM0kPyx3AoYK1MpRWO0T9V2ns=",
-    "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597",
-    "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf",
-    "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe",
-    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:8fdd792a626413502e68c195f2097352bdc6a0df694f7df350ed784741eb587e",
-    "zh:976bbaf268cb497400fd5b3c774d218f3933271864345f18deebe4dcbfcd6afa",
-    "zh:b21b78ca581f98f4cdb7a366b03ae9db23a73dfa7df12c533d7c19b68e9e72e5",
-    "zh:b7fc0c1615dbdb1d6fd4abb9c7dc7da286631f7ca2299fb9cd4664258ccfbff4",
-    "zh:d1efc942b2c44345e0c29bc976594cb7278c38cfb8897b344669eafbc3cddf46",
-    "zh:e356c245b3cd9d4789bab010893566acace682d7db877e52d40fc4ca34a50924",
-    "zh:ea98802ba92fcfa8cf12cbce2e9e7ebe999afbf8ed47fa45fc847a098d89468b",
-    "zh:eff8872458806499889f6927b5d954560f3d74bf20b6043409edf94d26cd906f",
-  ]
-}
diff --git a/deploy/terraform-datacommons-website/examples/website_v1/main.tf b/deploy/terraform-datacommons-website/examples/website_v1/main.tf
index 9e3b8392dd..8dbf3ebe57 100644
--- a/deploy/terraform-datacommons-website/examples/website_v1/main.tf
+++ b/deploy/terraform-datacommons-website/examples/website_v1/main.tf
@@ -64,7 +64,7 @@ module "apikeys" {
   source                   =  "../../modules/apikeys"
   project_id               = var.project_id
   dc_website_domain        = var.dc_website_domain
-  location                 = var.region
+  location                 = var.location
 
   resource_suffix          = local.resource_suffix
 }
@@ -78,7 +78,7 @@ module "esp" {
 module "cluster" {
   source                   =  "../../modules/gke"
   project_id               = var.project_id
-  region                   = var.region
+  location                 = var.location
   cluster_name_prefix      = var.cluster_name_prefix
   web_robot_sa_email       = local.web_robot_sa_email
 
@@ -99,48 +99,3 @@ resource "google_compute_managed_ssl_certificate" "dc_website_cert" {
     domains = [format("%s.", var.dc_website_domain)]
   }
 }
-
-# IMPORTANT NOTE: This script assumes that
-# "~/.kube/config" already exists. This is because provider cannot depend on data or resources,
-# as provider blocks need to be determined before resources/data states are fetched.
-# In install_custom_dc.sh, currentlythe kubeconfig is fetched before calling terraform apply.
-# .kube/config is the location where gcloud command for GKE stores cluster config, which
-# is required to access the cluster, including using helm.
-provider "kubernetes" {
-  alias = "datcom"
-  kubernetes {
-    config_path = "~/.kube/config"
-  }
-}
-
-provider "helm" {
-  alias = "datcom"
-  kubernetes {
-    config_path = "~/.kube/config"
-  }
-}
-
-module "k8s_resources" {
-  providers = {
-    kubernetes = kubernetes.datcom
-    helm       = helm.datcom
-  }
-
-  resource_suffix          =  local.resource_suffix
-  website_githash          =  var.website_githash
-  mixer_githash            =  var.mixer_githash
-
-  source                   =  "../../modules/helm"
-  project_id               = var.project_id
-
-  cluster_name             = module.cluster.name
-  cluster_region           = var.region
-  dc_website_domain        = var.dc_website_domain
-  global_static_ip_name    = format("dc-website-ip%s", local.resource_suffix)
-  managed_cert_name        = google_compute_managed_ssl_certificate.dc_website_cert.name
-
-  depends_on = [
-    google_compute_managed_ssl_certificate.dc_website_cert,
-    module.cluster
-  ]
-}
diff --git a/deploy/terraform-datacommons-website/examples/website_v1/outputs.tf b/deploy/terraform-datacommons-website/examples/website_v1/outputs.tf
index 6eb1179fcc..416e1a50f7 100644
--- a/deploy/terraform-datacommons-website/examples/website_v1/outputs.tf
+++ b/deploy/terraform-datacommons-website/examples/website_v1/outputs.tf
@@ -2,6 +2,6 @@ output "cluster_name" {
   value = module.cluster.name
 }
 
-output "cluster_region" {
-  value = module.cluster.region
+output "cluster_location" {
+  value = module.cluster.location
 }
diff --git a/deploy/terraform-datacommons-website/examples/website_v1/variables.tf b/deploy/terraform-datacommons-website/examples/website_v1/variables.tf
index c60b40b28c..2c5e1a2a2f 100644
--- a/deploy/terraform-datacommons-website/examples/website_v1/variables.tf
+++ b/deploy/terraform-datacommons-website/examples/website_v1/variables.tf
@@ -45,10 +45,21 @@ variable "cluster_name_prefix" {
   default     = "datacommons"
 }
 
-variable "region" {
+variable "location" {
   type        =  string
-  description = "GCP region where the cluster will be created in."
-  default     = "us-central1"
+  description = <<EOF
+Location of the GCP resources to be created.
+
+Can be regional, like "us-central1". Or zonal like "us-central1-a"
+
+Major difference between regional and zonal is that for GKE cluster, regional
+clusters will have nodes in each zone of that region, giving higher availability,
+but is more expensive.
+
+For regional only resources, if zonal location is specified, the region
+will be parsed from the zone.
+EOF
+  default     = "us-central1-a"
 }
 
 variable "global_static_ip_name" {
diff --git a/deploy/terraform-datacommons-website/modules/apikeys/main.tf b/deploy/terraform-datacommons-website/modules/apikeys/main.tf
index e69b79ee5e..f1f2296cb9 100644
--- a/deploy/terraform-datacommons-website/modules/apikeys/main.tf
+++ b/deploy/terraform-datacommons-website/modules/apikeys/main.tf
@@ -45,7 +45,8 @@ resource "google_secret_manager_secret" "maps_api_key_secret" {
   replication {
     user_managed {
       replicas {
-        location = var.location
+        # location needs to be a region here, so if var.location is a zone, make it a region.
+        location = length(split(var.location, "-")) == 2 ? var.location : join("-", slice(split( "-", var.location), 0, 2))
       }
     }
   }
diff --git a/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh b/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
index 15a8261094..bdd804fddc 100644
--- a/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
+++ b/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
@@ -12,8 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+if [[ $LOCATION =~ ^[a-z]+-[a-z0-9]+$ ]]; then
+  REGION=$LOCATION
+else
+  ZONE=$LOCATION
+fi
 gcloud container clusters get-credentials $CLUSTER_NAME \
-  --region $REGION --project=$PROJECT_ID
+  ${REGION:+--region $REGION} ${ZONE:+--zone $ZONE} --project=$PROJECT_ID
 
 # Create namespace if it does not exist.
 kubectl create namespace website \
diff --git a/deploy/terraform-datacommons-website/modules/gke/create_cluster.sh b/deploy/terraform-datacommons-website/modules/gke/create_cluster.sh
deleted file mode 100644
index ec83c5f03c..0000000000
--- a/deploy/terraform-datacommons-website/modules/gke/create_cluster.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-
-#!/bin/bash
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-gcloud container clusters create $CLUSTER_NAME \
-  --num-nodes=$NODES \
-  --region=$REGION \
-  --project=$PROJECT_ID \
-  --machine-type=e2-highmem-4 \
-  --enable-ip-alias \
-  --workload-pool=$PROJECT_ID.svc.id.goog \
-  --scopes=https://www.googleapis.com/auth/trace.append
diff --git a/deploy/terraform-datacommons-website/modules/gke/main.tf b/deploy/terraform-datacommons-website/modules/gke/main.tf
index 7f1ec0ff98..9b5cc5af6a 100644
--- a/deploy/terraform-datacommons-website/modules/gke/main.tf
+++ b/deploy/terraform-datacommons-website/modules/gke/main.tf
@@ -13,24 +13,64 @@
 # limitations under the License.
 
 locals {
-    # Example cluster name: datacommons-us-central1
-    cluster_name = format("%s-%s%s",var.cluster_name_prefix,var.region, var.resource_suffix)
+    # Example cluster name: datacommons-us-central1 or datacommons-us-central1-a
+    cluster_name = format("%s-%s%s",var.cluster_name_prefix,var.location, var.resource_suffix)
 }
 
-resource "null_resource" "gke_cluster" {
-  provisioner "local-exec" {
-    command = "sh create_cluster.sh"
-    working_dir = path.module
+# resource "null_resource" "gke_cluster" {
+#   provisioner "local-exec" {
+#     command = "sh create_cluster.sh"
+#     working_dir = path.module
 
-    environment = {
-      PROJECT_ID   = var.project_id
-      CLUSTER_NAME = local.cluster_name
-      NODES        = var.num_nodes
-      REGION       = var.region
-    }
+#     environment = {
+#       PROJECT_ID   = var.project_id
+#       CLUSTER_NAME = local.cluster_name
+#       NODES        = var.num_nodes
+#       REGION       = var.region
+#     }
+#   }
+# }
+
+resource "google_container_cluster" "primary" {
+  name     = local.cluster_name
+  location = var.location
+
+  # We can't create a cluster with no node pool defined, but we want to only use
+  # separately managed node pools. So we create the smallest possible default
+  # node pool and immediately delete it.
+  remove_default_node_pool = true
+  initial_node_count       = 1
+  networking_mode          = "VPC_NATIVE"
+
+  workload_identity_config {
+    workload_pool = "${var.project_id}.svc.id.goog"
+  }
+
+  ip_allocation_policy {
+    cluster_ipv4_cidr_block = "/14"
   }
 }
 
+resource "google_container_node_pool" "gke_node_pools" {
+  name       = local.cluster_name
+  location   = var.location
+  cluster    = google_container_cluster.primary.name
+  node_count = var.num_nodes
+
+  node_config {
+    machine_type = "e2-highmem-4"
+
+    # # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles.
+    # service_account = google_service_account.default.email
+    oauth_scopes    = [
+      "https://www.googleapis.com/auth/cloud-platform"
+    ]
+  }
+
+  depends_on = [
+    google_container_cluster.primary
+  ]
+}
 
 resource "null_resource" "gke_cluster_configuration" {
   provisioner "local-exec" {
@@ -40,12 +80,13 @@ resource "null_resource" "gke_cluster_configuration" {
     environment = {
       PROJECT_ID         = var.project_id
       CLUSTER_NAME       = local.cluster_name
-      REGION             = var.region
+      LOCATION           = var.location
       WEB_ROBOT_SA_EMAIL = var.web_robot_sa_email
     }
   }
 
   depends_on = [
-    null_resource.gke_cluster
+    google_container_cluster.primary,
+    google_container_node_pool.gke_node_pools
   ]
 }
diff --git a/deploy/terraform-datacommons-website/modules/gke/outputs.tf b/deploy/terraform-datacommons-website/modules/gke/outputs.tf
index d729d9a086..f4728f23a5 100644
--- a/deploy/terraform-datacommons-website/modules/gke/outputs.tf
+++ b/deploy/terraform-datacommons-website/modules/gke/outputs.tf
@@ -16,6 +16,6 @@ output "name" {
   value = local.cluster_name
 }
 
-output "region" {
-  value = var.region
+output "location" {
+  value = var.location
 }
diff --git a/deploy/terraform-datacommons-website/modules/gke/variables.tf b/deploy/terraform-datacommons-website/modules/gke/variables.tf
index 1511bafbca..1340b3d1bc 100644
--- a/deploy/terraform-datacommons-website/modules/gke/variables.tf
+++ b/deploy/terraform-datacommons-website/modules/gke/variables.tf
@@ -17,13 +17,11 @@ variable "project_id" {
   description = "GCP project id."
 }
 
-variable "region" {
+variable "location" {
   type        = string
-  description = "Region to create the GKE cluster in. Ex: us-central1"
-  default     = "us-central1"
+  description = "If location is a region like 'us-west1', then create a regional cluster. Otherwise create a zonal cluster."
 }
 
-
 variable "cluster_name_prefix" {
   type        = string
   description = "Prefix of the GKE cluster (to be used by the DC website) to create."
diff --git a/scripts/install_custom_dc.sh b/scripts/install_custom_dc.sh
index d59149b4ee..08b3e80b87 100755
--- a/scripts/install_custom_dc.sh
+++ b/scripts/install_custom_dc.sh
@@ -121,10 +121,6 @@ terraform init \
   -backend-config="bucket=$TF_STATE_BUCKET" \
   -backend-config="prefix=website_v1"
 
-gcloud container clusters get-credentials $(terraform output --raw cluster_name) \
-  --region  $(terraform output --raw cluster_region) \
-  --project $PROJECT_ID || true
-
 gsutil cp \
   gs://datcom-mixer-grpc/mixer-grpc/mixer-grpc.$MIXER_GITHASH.pb \
   $WEBSITE_ROOT/deploy/terraform-datacommons-website/modules/esp/mixer-grpc.$MIXER_GITHASH.pb
@@ -137,6 +133,52 @@ terraform apply \
   -var="mixer_githash=$MIXER_GITHASH" \
   -auto-approve
 
+# Install k8s resources using helm.
+CLUSTER_LOCATION=$(terraform output --raw cluster_location)
+if [[ $CLUSTER_LOCATION =~ ^[a-z]+-[a-z0-9]+$ ]]; then
+  REGION=$CLUSTER_LOCATION
+else
+  ZONE=$CLUSTER_LOCATION
+fi
+gcloud container clusters get-credentials $(terraform output --raw cluster_name) \
+  ${REGION:+--region $REGION} ${ZONE:+--zone $ZONE}  \
+  --project $PROJECT_ID || true
+
+cd $WEBSITE_ROOT
+
+if [[ -n "$WEBSITE_IMAGE_PROJECT_ID" ]]; then
+  echo "WEBSITE_IMAGE_PROJECT_ID is not set, using default: datcom-ci"
+  WEBSITE_IMAGE_PROJECT_ID="datcom-ci"
+fi
+
+if [[ -z "$MIXER_IMAGE_PROJECT_ID" ]]; then
+  echo "MIXER_IMAGE_PROJECT_ID is not set, using default: datcom-ci"
+  MIXER_IMAGE_PROJECT_ID="datcom-ci"
+fi
+
+helm upgrade --install \
+  dc-website deploy/helm_charts/dc_website \
+  --atomic \
+  --debug \
+  --timeout 10m \
+  --set resource_suffix=${RESOURCE_SUFFIX:-""} \
+  --set website.image.project="$WEBSITE_IMAGE_PROJECT_ID" \
+  --set website.image.tag="$WEBSITE_GITHASH" \
+  --set website.githash="$WEBSITE_GITHASH" \
+  --set mixer.image.project="$MIXER_IMAGE_PROJECT_ID" \
+  --set mixer.image.tag="$MIXER_GITHASH" \
+  --set mixer.githash="$MIXER_GITHASH" \
+  --set website.gcpProjectID="$PROJECT_ID" \
+  --set website.domain="$DOMAIN" \
+  --set website.secretGCPProjectID="$PROJECT_ID" \
+  --set mixer.gcpProjectID="$PROJECT_ID" \
+  --set mixer.serviceName="website-esp.endpoints.$PROJECT_ID.cloud.goog" \
+  --set ingress.enabled=true \
+  --set-file mixer.schemaConfigs."base\.mcf"=mixer/deploy/mapping/base.mcf \
+  --set-file mixer.schemaConfigs."encode\.mcf"=mixer/deploy/mapping/encode.mcf \
+  --set-file kgStoreConfig.bigqueryVersion=mixer/deploy/storage/bigquery.version \
+  --set-file kgStoreConfig.baseBigtableInfo=mixer/deploy/storage/base_bigtable_info.yaml
+
 # Run the BT automation Terraform script to set up BT loader.
 cd $ROOT
 

From 09978ec768efa3c0c32772ce255961faebb19f99 Mon Sep 17 00:00:00 2001
From: Alex Chen <alexyfchen@google.com>
Date: Tue, 28 Feb 2023 20:22:59 -0800
Subject: [PATCH 2/4] Remove some commented out code

---
 .../modules/gke/main.tf                       | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/deploy/terraform-datacommons-website/modules/gke/main.tf b/deploy/terraform-datacommons-website/modules/gke/main.tf
index 9b5cc5af6a..acda40883e 100644
--- a/deploy/terraform-datacommons-website/modules/gke/main.tf
+++ b/deploy/terraform-datacommons-website/modules/gke/main.tf
@@ -17,20 +17,6 @@ locals {
     cluster_name = format("%s-%s%s",var.cluster_name_prefix,var.location, var.resource_suffix)
 }
 
-# resource "null_resource" "gke_cluster" {
-#   provisioner "local-exec" {
-#     command = "sh create_cluster.sh"
-#     working_dir = path.module
-
-#     environment = {
-#       PROJECT_ID   = var.project_id
-#       CLUSTER_NAME = local.cluster_name
-#       NODES        = var.num_nodes
-#       REGION       = var.region
-#     }
-#   }
-# }
-
 resource "google_container_cluster" "primary" {
   name     = local.cluster_name
   location = var.location
@@ -59,10 +45,7 @@ resource "google_container_node_pool" "gke_node_pools" {
 
   node_config {
     machine_type = "e2-highmem-4"
-
-    # # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles.
-    # service_account = google_service_account.default.email
-    oauth_scopes    = [
+    oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
   }

From 74a6827bc8034bb4e90b0e44b21323a85968c852 Mon Sep 17 00:00:00 2001
From: Alex Chen <alexyfchen@google.com>
Date: Wed, 1 Mar 2023 15:17:35 -0800
Subject: [PATCH 3/4] PR comments

---
 .../helm_charts/dc_website/templates/deployment.yaml | 12 ++++++------
 deploy/helm_charts/dc_website/values.yaml            |  5 -----
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/deploy/helm_charts/dc_website/templates/deployment.yaml b/deploy/helm_charts/dc_website/templates/deployment.yaml
index a94d76b337..7beec761a2 100644
--- a/deploy/helm_charts/dc_website/templates/deployment.yaml
+++ b/deploy/helm_charts/dc_website/templates/deployment.yaml
@@ -56,7 +56,7 @@ spec:
             name: schema-mapping{{ .Values.resourceSuffix }}
       containers:
         - name: website
-          image:  "{{ .Values.website.image.registry }}/{{ .Values.website.image.project }}/{{ .Values.website.image.repository }}:{{ .Values.website.image.tag }}"
+          image:  "gcr.io/{{ .Values.website.image.project }}/datacommons-website:{{ .Values.website.image.tag }}"
           imagePullPolicy: {{ .Values.website.image.pullPolicy }}
           args: []
           ports:
@@ -75,9 +75,9 @@ spec:
             periodSeconds: 10
           resources:
             limits:
-              memory: "8G"
+              memory: "3G"
             requests:
-              memory: "8G"
+              memory: "3G"
           volumeMounts:
             - name: ai-config
               mountPath: /datacommons/ai
@@ -117,7 +117,7 @@ spec:
             - name: BOUNCE
               value: "dummy"
         - name: mixer
-          image:  "{{ .Values.mixer.image.registry }}/{{ .Values.mixer.image.project }}/{{ .Values.mixer.image.repository }}:{{ .Values.mixer.image.tag }}"
+          image:  "gcr.io/{{ .Values.mixer.image.project }}/datacommons-mixer:{{ .Values.mixer.image.tag }}"
           imagePullPolicy: {{ .Values.mixer.image.pullPolicy }}
           resources:
             limits:
@@ -201,9 +201,9 @@ spec:
                   key: serviceName
           resources:
             limits:
-              memory: "2G"
+              memory: "1G"
             requests:
-              memory: "2G"
+              memory: "1G"
           readinessProbe:
             httpGet:
               path: /healthz
diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml
index c1e0d148d5..8824f6c560 100644
--- a/deploy/helm_charts/dc_website/values.yaml
+++ b/deploy/helm_charts/dc_website/values.yaml
@@ -13,9 +13,7 @@ website:
   githash:
 
   image:
-    registry: "gcr.io"
     project: "datcom-ci"
-    repository: "datacommons-website"
     pullPolicy: Always
     tag:
 
@@ -56,10 +54,7 @@ mixer:
   githash:
 
   image:
-    registry: "gcr.io"
     project: "datcom-ci"
-    repository: "datacommons-mixer"
-
     pullPolicy: Always
     tag:
 

From b5a47e04988b35a1a2a2b6b021da37cd88ae29a6 Mon Sep 17 00:00:00 2001
From: Alex Chen <alexyfchen@google.com>
Date: Wed, 1 Mar 2023 19:53:01 -0800
Subject: [PATCH 4/4] PR comment

---
 .../modules/gke/configure_cluster.sh                             | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh b/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
index bdd804fddc..014fc5fffb 100644
--- a/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
+++ b/deploy/terraform-datacommons-website/modules/gke/configure_cluster.sh
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Region is like "us-central1", zone is like "us-central1-a"
 if [[ $LOCATION =~ ^[a-z]+-[a-z0-9]+$ ]]; then
   REGION=$LOCATION
 else