infra/gcp: clean stale removal code, misc prow cleanup

infra/gcp/ensure-gsuite.sh

@@ -71,10 +71,10 @@ ensure_service_account \
     "Grants access to the googlegroups API in kubernetes.io GSuite"
 
 # Allow k8s-infra-prow-builds-trusted to run pods as this service account
-color 6 "Empowering ${GSUITE_SVCACCT} to be used on k8s-infra-prow-build-trusted"
-empower_ksa_to_svcacct \
-    "k8s-infra-prow-build-trusted.svc.id.goog[test-pods/${GSUITE_SVCACCT}]" \
-    "${PROJECT}" \
+color 6 "Ensuring GKE clusters in 'k8s-infra-prow-build-trusted' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${GSUITE_SVCACCT}'"
+empower_gke_for_serviceaccount \
+    "k8s-infra-prow-build-trusted" \
+    "${PROWJOB_POD_NAMESPACE}" \
     "$(svc_acct_email "${PROJECT}" "${GSUITE_SVCACCT}")"
 
 # Ensure the service account has a key in a secret accessible by the right people

infra/gcp/ensure-main-project.sh

@@ -343,21 +343,12 @@ function ensure_main_project() {
         # is custom role audit.viewer on the kubernetes.io org, but that is
         # handled by ensure-organization.sh
         svcacct_args=("${project}" "k8s-infra-gcp-auditor" "roles/viewer")
-        cluster_args=("k8s-infra-prow-build-trusted" "test-pods")
+        cluster_args=("k8s-infra-prow-build-trusted" "${PROWJOB_POD_NAMESPACE}")
         ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
 
-        # TODO(spiffxp): remove once this binding has been deleted
-        local gcp_auditor_email
-        gcp_auditor_email=$(svc_acct_email "${project}" "k8s-infra-gcp-auditor")
-        color 6 "Ensuring removed workload identity on kubernetes-public/test-pods for ${gcp_auditor_email}"
-        unempower_gke_for_serviceaccount \
-            "kubernetes-public" \
-            "test-pods" \
-            "${gcp_auditor_email}" 2>&1 | indent
-
         color 6 "Ensuring DNS Updater serviceaccount"
         svcacct_args=("${project}" "k8s-infra-dns-updater" "roles/dns.admin")
-        cluster_args=("k8s-infra-prow-build-trusted" "test-pods")
+        cluster_args=("k8s-infra-prow-build-trusted" "${PROWJOB_POD_NAMESPACE}")
         ensure_workload_identity_serviceaccount "${svcacct_args[@]}" "${cluster_args[@]}" 2>&1 | indent
 
         color 6 "Ensuring Monitoring Viewer serviceaccount"

infra/gcp/ensure-organization.sh

@@ -71,21 +71,7 @@ org_role_bindings=(
   "serviceAccount:$(svc_acct_email "kubernetes-public" "k8s-infra-gcp-auditor"):$(custom_org_role_name "audit.viewer")"
 )
 
-removed_org_role_bindings=(
-  # TODO(spiffxp): remove all of these in followup PR once deployed
-  "group:k8s-infra-gcp-auditors@kubernetes.io:roles/secretmanager.viewer"
-  "user:davanum@gmail.com:roles/compute.viewer"
-  "user:davanum@gmail.com:roles/dns.reader"
-  "user:davanum@gmail.com:roles/iam.securityReviewer"
-  "user:davanum@gmail.com:roles/resourcemanager.organizationViewer"
-  "user:davanum@gmail.com:roles/serviceusage.serviceUsageConsumer"
-  "user:thockin@google.com:roles/compute.viewer"
-  "user:thockin@google.com:roles/dns.reader"
-  "user:thockin@google.com:roles/iam.securityReviewer"
-  "user:thockin@google.com:roles/resourcemanager.organizationViewer"
-  "user:thockin@google.com:roles/serviceusage.serviceUsageConsumer"
-  "user:spiffxp@google.com:roles/resourcemanager.organizationAdmin"
-)
+removed_org_role_bindings=()
 
 function ensure_org_roles() {
     for role in "${org_roles[@]}"; do

infra/gcp/ensure-prod-storage.sh

@@ -225,6 +225,8 @@ function ensure_all_prod_buckets() {
 
 
 function ensure_all_prod_special_cases() {
+    local serviceaccount
+
     # Special case: set the web policy on the prod bucket.
     color 6 "Configuring the web policy on the prod bucket"
     ensure_gcs_web_policy "gs://${PROD_PROJECT}"
@@ -324,61 +326,62 @@ function ensure_all_prod_special_cases() {
         "${PROD_PROJECT}" \
         "k8s-infra-artifact-admins@kubernetes.io"
 
-    # Special case: empower Kubernetes service account to authenticate as a GCP
-    # service account.
-    #
-    # For write access to k8s-artifacts-prod GCR.
-    color 6 "Empowering promoter namespace to use prod promoter svcacct"
-    for project in "${PROW_TRUSTED_BUILD_CLUSTER_PROJECTS[@]}"; do
-        empower_ksa_to_svcacct \
-            "${project}.svc.id.goog[test-pods/k8s-infra-gcr-promoter]" \
-            "${PROD_PROJECT}" \
-            "$(svc_acct_email "${PROD_PROJECT}" "${PROMOTER_SVCACCT}")"
-    done
-    # For write access to k8s-artifacts-prod-bak GCR. This is only for backups.
-    color 6 "Empowering promoter-bak namespace to use prod-bak promoter svcacct"
+    # TODO: what is this used for?
+    color 6 "Ensuring prod promoter vuln scanning svcacct exists"
+    ensure_service_account \
+        "${PROD_PROJECT}" \
+        "${PROMOTER_VULN_SCANNING_SVCACCT}" \
+        "k8s-infra container image vuln scanning"
+
+    # Special case: allow prow trusted build clusters to run jobs as
+    # prod-related GCP service accounts
+    color 6 "Empowering trusted prow build clusters to use prod-related GCP service accounts"
     for project in "${PROW_TRUSTED_BUILD_CLUSTER_PROJECTS[@]}"; do
-        empower_ksa_to_svcacct \
-            "${project}.svc.id.goog[test-pods/k8s-infra-gcr-promoter-bak]" \
-            "${PRODBAK_PROJECT}" \
-            "$(svc_acct_email "${PRODBAK_PROJECT}" "${PROMOTER_SVCACCT}")"
+        # Grant write access to k8s-artifacts-prod GCR
+        serviceaccount="$(svc_acct_email "${PROD_PROJECT}" "${PROMOTER_SVCACCT}")"
+        color 6 "Ensuring GKE clusters in '${project}' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${serviceaccount}'"
+        empower_gke_for_serviceaccount \
+            "${project}" "${PROWJOB_POD_NAMESPACE}" \
+            "${serviceaccount}" "k8s-infra-gcr-promoter"
+
+        # Grant write access to k8s-artifacts-prod-bak GCR (for backups)
+        serviceaccount="$(svc_acct_email "${PRODBAK_PROJECT}" "${PROMOTER_SVCACCT}")"
+        color 6 "Ensuring GKE clusters in '${project}' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${serviceaccount}'"
+        empower_gke_for_serviceaccount \
+            "${project}" "${PROWJOB_POD_NAMESPACE}" \
+            "${serviceaccount}" "k8s-infra-gcr-promoter-bak"
+
+        # TODO: Grant ??? acccess to k8s-artifacts-prod ???
+        serviceaccount="$(svc_acct_email "${PROD_PROJECT}" "${PROMOTER_VULN_SCANNING_SVCACCT}")"
+        color 6 "Ensuring GKE clusters in '${project}' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${serviceaccount}'"
+        empower_gke_for_serviceaccount \
+            "${project}" "${PROWJOB_POD_NAMESPACE}" \
+            "${serviceaccount}" "k8s-infra-gcr-vuln-scanning" 
     done
+
     # For write access to:
     #   (1) k8s-gcr-backup-test-prod GCR
     #   (2) k8s-gcr-backup-test-prod-bak GCR.
     # Even though we only grant authentication to 1 SA
     # (k8s-infra-gcr-promoter@k8s-gcr-backup-test-prod-bak.iam.gserviceaccount.com),
     # this SA has write access to the above 2 GCRs, fulfilling our needs.
     #
-    # Also, note that the project name for the GKE cluster is "k8s-prow-builds",
-    # which is the non-trusted Prow cluster.
+    # NOTE: This is granted to prow build clusters that run untrusted code,
+    #       such as presubmit jobs using PRs.
     color 6 "Empowering promoter-test namespace to use backup-test-prod-bak promoter svcacct"
+    serviceaccount="$(svc_acct_email "${GCR_BACKUP_TEST_PRODBAK_PROJECT}" "${PROMOTER_SVCACCT}")"
     for project in "${PROW_UNTRUSTED_BUILD_CLUSTER_PROJECTS[@]}"; do
-        empower_ksa_to_svcacct \
-            "${project}.svc.id.goog[test-pods/k8s-infra-gcr-promoter-test]" \
-            "${GCR_BACKUP_TEST_PRODBAK_PROJECT}" \
-            "$(svc_acct_email "${GCR_BACKUP_TEST_PRODBAK_PROJECT}" "${PROMOTER_SVCACCT}")"
+        color 6 "Ensuring GKE clusters in '${project}' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${serviceaccount}'"
+        empower_gke_for_serviceaccount \
+            "${project}" "${PROWJOB_POD_NAMESPACE}" \
+            "${serviceaccount}" "k8s-infra-gcr-promoter-test"
     done
 
     # Special case: empower k8s-infra-gcs-access-logs@kubernetes.io to read k8s-artifacts-gcslogs
     # k8s-artifacts-gcslogs receive and store Cloud Audit logs for k8s-artificats-prod.
     ensure_gcs_role_binding "gs://k8s-artifacts-gcslogs" \
         "group:k8s-infra-gcs-access-logs@kubernetes.io" \
         "objectViewer"
-
-    color 6 "Ensuring prod promoter vuln scanning svcacct exists"
-    ensure_service_account \
-        "${PROD_PROJECT}" \
-        "${PROMOTER_VULN_SCANNING_SVCACCT}" \
-        "k8s-infra container image vuln scanning"
-
-    color 6 "Empowering promoter-scanning namespace to use prod promoter vuln scanning svcacct"
-    for project in "${PROW_TRUSTED_BUILD_CLUSTER_PROJECTS[@]}"; do
-        empower_ksa_to_svcacct \
-            "${project}.svc.id.goog[test-pods/k8s-infra-gcr-vuln-scanning]" \
-            "${PROD_PROJECT}" \
-            "$(svc_acct_email "${PROD_PROJECT}" "${PROMOTER_VULN_SCANNING_SVCACCT}")"
-    done
 }
 
 function main() {

infra/gcp/ensure-release-projects.sh

@@ -132,9 +132,6 @@ for PROJECT; do
     ensure_gcs_role_binding "${GCB_BUCKET}" "${principal}" "objectCreator"
     ensure_gcs_role_binding "${GCB_BUCKET}" "${principal}" "objectViewer"
 
-    color 6 "Ensuring k8s-prow / test-infra-trusted can no longer use GCB in project: ${PROJECT}"
-    ensure_removed_google_prow_bindings "${PROJECT}" "${GCB_BUCKET}"
-
     # Let project admins use KMS.
     color 6 "Empowering ${RELEASE_ADMINS} as KMS admins"
     empower_group_for_kms "${PROJECT}" "${RELEASE_ADMINS}"

infra/gcp/ensure-staging-storage.sh

@@ -318,18 +318,15 @@ function ensure_staging_gcb() {
     ensure_project_role_binding "${project}" "${principal}" "roles/cloudbuild.builds.builder"
     ensure_gcs_role_binding "${bucket}" "${principal}" "objectCreator"
     ensure_gcs_role_binding "${bucket}" "${principal}" "objectViewer"
-
-    color 6 "Ensuring k8s-prow / test-infra-trusted can no longer use GCB in project: ${project}"
-    ensure_removed_google_prow_bindings "${project}" "${bucket}"
 }
 
 # TODO(spiffxp): rename this to just prow@project and deprecate/rm the gcb-builder-foo
 #                serviceaccounts; this will allow prowjobs to write to GCS or GCR in
 #                the project directly, as well as triggering GCB to do the same
 # Create a gcb-builder-{staging} GCP service account in project k8s-staging-{staging}
 # that can trigger GCB within that project. Allow GKE clusters in {prow_project}
-# to use this when running as a kubernetes service account of the same name in
-# the "test-pods" namespace
+# to use this when running pods as a kubernetes service account of the same name in
+# PROWJOB_POD_NAMESPACE
 #
 # $1: The staging name (e.g. kubetest2)
 # $2: The prow project name (e.g. k8s-infra-prow-build)
@@ -341,7 +338,6 @@ function ensure_staging_gcb_builder_service_account() {
 
     local staging="$1"
     local prow_project="$2"
-    local prow_job_namespace="test-pods"
     local project="k8s-staging-${staging}"
     local sa_name="gcb-builder-${staging}"
     local sa_email="${sa_name}@${project}.iam.gserviceaccount.com"
@@ -364,10 +360,10 @@ function ensure_staging_gcb_builder_service_account() {
     ensure_gcs_role_binding "${gcb_bucket}" "${principal}" "objectCreator"
     ensure_gcs_role_binding "${gcb_bucket}" "${principal}" "objectViewer"
 
-    color 6 "Ensuring ${sa_email} usable by GKE clusters in ${prow_project} running as ${sa_name} in ${prow_job_namespace} namespace"
-    empower_ksa_to_svcacct \
-        "${prow_project}.svc.id.goog[${prow_job_namespace}/${sa_name}]" \
-        "${project}" \
+    color 6 "Ensuring GKE clusters in '${prow_project}' can run pods in '${PROWJOB_POD_NAMESPACE}' as '${sa_email}'"
+    empower_gke_for_serviceaccount \
+        "${prow_project}" \
+        "${PROWJOB_POD_NAMESPACE}" \
         "${sa_email}"
 }
 

infra/gcp/lib.sh

@@ -97,15 +97,11 @@ readonly GCB_BUILDER_SVCACCT="gcb-builder@k8s-infra-prow-build-trusted.iam.gserv
 
 readonly PROW_BUILD_SERVICE_ACCOUNT="prow-build@k8s-infra-prow-build.iam.gserviceaccount.com"
 
-# TODO: decommission this once we've flipped to prow-build-trusted
-# The service account email for Prow (not in this org for now).
-readonly PROW_GOOGLE_TRUSTED_SERVICE_ACCOUNT="deployer@k8s-prow.iam.gserviceaccount.com"
-
 # Projects hosting prow build clusters that run untrusted code, such as
 # presubmits that build and test unmerged code from PRs
 readonly PROW_UNTRUSTED_BUILD_CLUSTER_PROJECTS=(
     # The google.com build cluster for prow.k8s.io
-    # TODO: remove support for this where possible
+    # TODO(spiffxp): remove support for this where possible
     "k8s-prow-builds"
     # The kubernetes.io build cluster
     "k8s-infra-prow-build"
@@ -115,12 +111,22 @@ readonly PROW_UNTRUSTED_BUILD_CLUSTER_PROJECTS=(
 # that run merged/approved code that need access to sensitive secrets
 readonly PROW_TRUSTED_BUILD_CLUSTER_PROJECTS=(
     # The google.com trusted build cluster for prow.k8s.io
-    # TODO: remove support for this where possible
+    # TODO(spiffxp): remove support for this where possible
     "k8s-prow"
     # The kubernetes.io build cluster
     "k8s-infra-prow-build-trusted"
 )
 
+# The namespace prowjobs run in; at present things are configured to use the
+# same namespace across all prow build clusters. This means this value needs
+# to be kept consistent across a few places:
+#
+# - https://git.k8s.io/test-infra/config/prow/config.yaml # pod_namespace: test-pods
+# - infra/gcp/clusters/projects/k8s-infra-prow-*/*/main.tf # pod_namespace = test-pods
+# # TODO: not all resources belong in test-pods, would be good to shard into folders
+# - infra/gcp/clusters/projects/k8s-infra-prow-*/*/resources/* # namespace: test-pods
+readonly PROWJOB_POD_NAMESPACE="test-pods"
+
 #
 # Functions
 #
@@ -298,26 +304,6 @@ function empower_group_for_kms() {
     done
 }
 
-# TODO(spiffxp): remove this in a follow-up PR
-# Remove privileges previously granted to the google-owned test-infra-trusted
-# cluster's "deployer" service account
-# $1: The GCP project
-# $2: The GCS scratch bucket
-function ensure_removed_google_prow_bindings() {
-    if [ $# -lt 2 ] || [ -z "$1" ] || [ -z "$2" ]; then
-        echo "${FUNCNAME[0]}(project, bucket) requires 2 arguments" >&2
-        return 1
-    fi
-    local project="$1"
-    local bucket="$2"
-
-    local google_prow_principal="serviceAccount:${PROW_GOOGLE_TRUSTED_SERVICE_ACCOUNT}"
-
-    ensure_removed_project_role_binding "${project}" "${google_prow_principal}" "roles/cloudbuild.builds.builder"
-    ensure_removed_gcs_role_binding "${bucket}" "${google_prow_principal}" "objectCreator"
-    ensure_removed_gcs_role_binding "${bucket}" "${google_prow_principal}" "objectViewer"
-}
-
 # Grant full privileges to GCR admins
 # $1: The GCP project
 # $2: The GCR region (optional)
@@ -495,32 +481,6 @@ function ensure_dns_zone() {
   fi
 }
 
-# Allow a Kubernetes service account (KSA) the ability to authenticate as the as
-# the given GCP service account for the given GKE Project's Kubernetes
-# namespace; this is also called Workload Identity.
-#
-# $1: The service account scoped to a (1) GKE project, (2) K8s namespace, and
-#     (3) K8s service account. The format is currently
-#
-#       "${gke_project}.svc.id.goog[${k8s_namespace}/${k8s_svcacct}]"
-#
-#     This is used to scope the grant of permissions to the combination of the
-#     above 3 variables
-# $2: The GCP project that owns the GCP service account.
-# $3: The GCP service account to draw powers from.
-function empower_ksa_to_svcacct() {
-    if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
-        echo "empower_ksa_to_svcacct(ksa_scope, gcp_project, gcp_scvacct) requires 3 arguments" >&2
-        return 1
-    fi
-
-    local ksa_scope="$1"
-    local gcp_project="$2"
-    local gcp_svcacct="$3"
-
-    ensure_serviceaccount_role_binding "${gcp_svcacct}" "serviceAccount:${ksa_scope}" "roles/iam.workloadIdentityUser"
-}
-
 # Allow GKE clusters in the given GCP project to run workloads using a
 # Kubernetes service account in the given namepsace to act as the given
 # GCP service account via Workload Identity when the name of the Kubernetes
@@ -532,7 +492,7 @@ function empower_ksa_to_svcacct() {
 # $1:   The GCP project that hosts the GKE clusters (e.g. k8s-infra-foo-clusters)
 # $2:   The K8s namespace that hosts the Kubernetes service account (e.g. my-app-ns)
 # $3:   The GCP service account to be bound (e.g. k8s-infra-doer@k8s-infra-foo.iam.gserviceaccount.com)
-# [$4]: Optional: The Kubernetes service account name (e.g. my-app-doer; default: k8s-infra-doer)
+# [$4]: Optional: The Kubernetes service account name (e.g. my-app-doer; default e.g. k8s-infra-doer)
 #
 # e.g. the above allows pods running as my-app-ns/my-app-doer in clusters in
 #      k8s-infra-foo-clusters to act as k8s-infra-doer@k8s-infra-foo.iam.gserviceaccount.com