Merge pull request #2089 from spiffxp/audit-no-etags

audit: strip etags from json output, fix bq access dump
kubernetes · May 24, 2021 · e13012c · e13012c
2 parents 7a4ee75 + 0e2e3d3
commit e13012c
Show file tree

Hide file tree

Showing 37 changed files with 307 additions and 57 deletions.
diff --git a/audit/audit-gcp.sh b/audit/audit-gcp.sh
@@ -20,6 +20,11 @@ set -o pipefail
 
 CNCF_GCP_ORG=758905017065
 
+function format_gcloud_json() {
+  # recursively delete any fields named "etag"
+  jq 'delpaths([path(..|.etag?|select(.))])'
+}
+
 echo "# Removing existing audit files"
 rm -rf org_kubernetes.io
 rm -rf projects
@@ -34,14 +39,12 @@ gcloud \
     ROLE=$(basename "${ROLE_PATH}")
     gcloud iam roles describe "${ROLE}" \
         --organization="${CNCF_GCP_ORG}" \
-        --format=json \
-        | jq 'del(.etag)' \
+        --format=json | format_gcloud_json \
         > "org_kubernetes.io/roles/${ROLE}.json"
 done
 gcloud \
     organizations get-iam-policy "${CNCF_GCP_ORG}" \
-    --format=json \
-    | jq 'del(.etag)' \
+    --format=json | format_gcloud_json \
     > "org_kubernetes.io/iam.json"
 
 echo "## Iterating over Projects"
@@ -55,16 +58,16 @@ gcloud \
 
     echo "### Auditing Project ${PROJECT}"
     mkdir -p "projects/${PROJECT}"
+
     gcloud \
         projects describe "${PROJECT}" \
-        --format=json \
+        --format=json | format_gcloud_json \
         > "projects/${PROJECT}/description.json"
 
     echo "#### ${PROJECT} IAM"
     gcloud \
         projects get-iam-policy "${PROJECT}" \
-        --format=json \
-        | jq 'del(.etag)' \
+        --format=json | format_gcloud_json \
         > "projects/${PROJECT}/iam.json"
 
     echo "#### ${PROJECT} ServiceAccounts"
@@ -77,14 +80,12 @@ gcloud \
         gcloud \
             iam service-accounts describe "${SVCACCT}" \
             --project="${PROJECT}" \
-            --format=json \
-            | jq 'del(.etag)' \
+            --format=json | format_gcloud_json \
             > "projects/${PROJECT}/service-accounts/${SVCACCT}/description.json"
         gcloud \
             iam service-accounts get-iam-policy "${SVCACCT}" \
             --project="${PROJECT}" \
-            --format=json \
-            | jq 'del(.etag)' \
+            --format=json | format_gcloud_json \
             > "projects/${PROJECT}/service-accounts/${SVCACCT}/iam.json"
     done
 
@@ -99,8 +100,7 @@ gcloud \
         gcloud \
             iam roles describe "${ROLE}" \
             --project="${PROJECT}" \
-            --format=json \
-            | jq 'del(.etag)' \
+            --format=json | format_gcloud_json \
             > "projects/${PROJECT}/roles/${ROLE}.json"
     done
 
@@ -116,22 +116,32 @@ gcloud \
         --format="value(config.name)" \
     | sed 's/.googleapis.com//' \
     | while read -r SVC; do
+        echo "##### projects/${PROJECT}/services/${SVC}"
         case "${SVC}" in
             bigquery)
                 mkdir -p "projects/${PROJECT}/services/${SVC}"
                 bq \
-                    --format=prettyjson --project_id=$PROJECT ls
+                    ls \
+                    --project_id="${PROJECT}" \
+                    --format=json | format_gcloud_json \
                     > "projects/${PROJECT}/services/${SVC}/bigquery.datasets.json"                
                 # Only run if there are any datasets
                 if [ -s "projects/${PROJECT}/services/${SVC}/bigquery.datasets.json" ]
                 then
                     bq \
-                        --project_id="{$PROJECT}" --format=json ls \
+                        ls \
+                        --project_id="${PROJECT}" \
+                        --format=json | format_gcloud_json \
                         | jq -r '.[] | .datasetReference["datasetId"]' \
                         | while read -r DATASET; do                        
                             bq \
-                                --project_id="${PROJECT}" --format=json show "${PROJECT}:${DATASET}" \
-                            | jq .access > "projects/${PROJECT}/services/${SVC}/bigquery.datasets.${DATASET}.access.json"
+                                show \
+                                --project_id="${PROJECT}" \
+                                --format=json \
+                                "${PROJECT}:${DATASET}" \
+                                | format_gcloud_json \
+                                | jq .access \
+                                > "projects/${PROJECT}/services/${SVC}/bigquery.datasets.${DATASET}.access.json"
                         done
                 fi
                 ;;
@@ -140,7 +150,7 @@ gcloud \
                 gcloud \
                     compute project-info describe \
                     --project="${PROJECT}" \
-                    --format=json \
+                    --format=json | format_gcloud_json \
                     | jq 'del(.quotas[].usage, .commonInstanceMetadata.fingerprint)' \
                     > "projects/${PROJECT}/services/${SVC}/project-info.json"
                 ;;
@@ -157,11 +167,11 @@ gcloud \
                 mkdir -p "projects/${PROJECT}/services/${SVC}"
                 gcloud \
                     dns project-info describe "${PROJECT}" \
-                    --format=json \
+                    --format=json | format_gcloud_json \
                     > "projects/${PROJECT}/services/${SVC}/info.json"
                 gcloud \
                     dns managed-zones list \
-                    --format=json \
+                    --format=json | format_gcloud_json \
                     > "projects/${PROJECT}/services/${SVC}/zones.json"
                 ;;
             logging)
@@ -187,7 +197,7 @@ gcloud \
                     gcloud \
                         secrets describe "${SECRET}" \
                         --project="${PROJECT}" \
-                        --format=json \
+                        --format=json | format_gcloud_json \
                         > "${path}/description.json"
                     gcloud \
                         secrets versions list "${SECRET}" \
@@ -197,8 +207,7 @@ gcloud \
                     gcloud \
                         secrets get-iam-policy "${SECRET}" \
                         --project="${PROJECT}" \
-                        --format=json \
-                        | jq 'del(.etag)' \
+                        --format=json | format_gcloud_json \
                         > "${path}/iam.json"
                 done
                 ;;
@@ -214,12 +223,12 @@ gcloud \
                     gsutil logging get "gs://${BUCKET}/" \
                         > "projects/${PROJECT}/buckets/${BUCKET}/logging.txt"
                     gsutil iam get "gs://${BUCKET}/" \
-                        | jq 'del(.etag)' \
+                        | format_gcloud_json \
                         > "projects/${PROJECT}/buckets/${BUCKET}/iam.json"
                 done
                 ;;
             *)
-                echo "##### Unhandled Service ${SVC}"
+                echo "WARN: Unaudited service enabled in project ${PROJECT}: ${SVC}"
                 # (these were all enabled for kubernetes-public)
                 # TODO: handle (or ignore) bigquerystorage
                 # TODO: handle (or ignore) clouderrorreporting
@@ -239,7 +248,3 @@ gcloud \
         esac
     done
 done
-
-
-# TODO:
-# Dump iam for Big Query
diff --git a/audit/projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.gcs_logs.access.json b/audit/projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.gcs_logs.access.json
@@ -0,0 +1,18 @@
+[
+  {
+    "role": "WRITER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectOwners"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "justinsb@google.com"
+  },
+  {
+    "role": "READER",
+    "specialGroup": "projectReaders"
+  }
+]
diff --git a/.../projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.http_lb_logs.access.json b/.../projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.http_lb_logs.access.json
@@ -0,0 +1,14 @@
+[
+  {
+    "role": "WRITER",
+    "userByEmail": "p388270116193-330742@gcp-sa-logging.iam.gserviceaccount.com"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "justinsb@google.com"
+  }
+]
diff --git a/audit/projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.json b/audit/projects/k8s-artifacts-prod/services/bigquery/bigquery.datasets.json
@@ -0,0 +1,20 @@
+[
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-artifacts-prod:gcs_logs",
+    "datasetReference": {
+      "datasetId": "gcs_logs",
+      "projectId": "k8s-artifacts-prod"
+    },
+    "location": "US"
+  },
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-artifacts-prod:http_lb_logs",
+    "datasetReference": {
+      "datasetId": "http_lb_logs",
+      "projectId": "k8s-artifacts-prod"
+    },
+    "location": "US"
+  }
+]
diff --git a/audit/projects/k8s-conform/secrets/service-cri-o-key/description.json b/audit/projects/k8s-conform/secrets/service-cri-o-key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2021-04-08T20:32:11.215176Z",
-  "etag": "\"15bf7bf125b148\"",
   "name": "projects/228988630781/secrets/service-cri-o-key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-conform/secrets/service-huaweicloud-key/description.json b/audit/projects/k8s-conform/secrets/service-huaweicloud-key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2021-04-08T20:43:10.411934Z",
-  "etag": "\"15bf7c18703c9e\"",
   "name": "projects/228988630781/secrets/service-huaweicloud-key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-conform/secrets/service-inspur-key/description.json b/audit/projects/k8s-conform/secrets/service-inspur-key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2021-02-23T06:37:04.961097Z",
-  "etag": "\"15bbfb25906e49\"",
   "name": "projects/228988630781/secrets/service-inspur-key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-conform/secrets/service-provider-openstack-key/description.json b/audit/projects/k8s-conform/secrets/service-provider-openstack-key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2021-02-15T15:18:08.840992Z",
-  "etag": "\"15bb617e4e6120\"",
   "name": "projects/228988630781/secrets/service-provider-openstack-key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-conform/secrets/service-s390x-k8s-key/description.json b/audit/projects/k8s-conform/secrets/service-s390x-k8s-key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2020-09-23T21:18:11.941957Z",
-  "etag": "\"15b3ed7a94947b\"",
   "name": "projects/228988630781/secrets/service-s390x-k8s-key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-gsuite/secrets/gsuite-groups-manager_key/description.json b/audit/projects/k8s-gsuite/secrets/gsuite-groups-manager_key/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2020-04-30T04:24:22.976608Z",
-  "etag": "\"15b3ed7c79f8c0\"",
   "name": "projects/91610859379/secrets/gsuite-groups-manager_key",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-gsuite/secrets/wg-k8s-infra-billing_pw/description.json b/audit/projects/k8s-gsuite/secrets/wg-k8s-infra-billing_pw/description.json
@@ -1,6 +1,5 @@
 {
   "createTime": "2020-05-11T16:52:59.141275Z",
-  "etag": "\"15b3ed7b29f480\"",
   "name": "projects/91610859379/secrets/wg-k8s-infra-billing_pw",
   "replication": {
     "automatic": {}

diff --git a/audit/projects/k8s-infra-ii-sandbox/services/bigquery/bigquery.datasets.hh.access.json b/audit/projects/k8s-infra-ii-sandbox/services/bigquery/bigquery.datasets.hh.access.json
@@ -0,0 +1,18 @@
+[
+  {
+    "role": "WRITER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectOwners"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "hh@ii.coop"
+  },
+  {
+    "role": "READER",
+    "specialGroup": "projectReaders"
+  }
+]
diff --git a/audit/projects/k8s-infra-ii-sandbox/services/bigquery/bigquery.datasets.json b/audit/projects/k8s-infra-ii-sandbox/services/bigquery/bigquery.datasets.json
@@ -0,0 +1,41 @@
+[
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-infra-ii-sandbox:hh",
+    "datasetReference": {
+      "datasetId": "hh",
+      "projectId": "k8s-infra-ii-sandbox"
+    },
+    "location": "US"
+  },
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-infra-ii-sandbox:k8s_artifacts_dataset_bb_test",
+    "datasetReference": {
+      "datasetId": "k8s_artifacts_dataset_bb_test",
+      "projectId": "k8s-infra-ii-sandbox"
+    },
+    "labels": {
+      "managed-by-cnrm": "true"
+    },
+    "location": "US"
+  },
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-infra-ii-sandbox:k8s_artifacts_gcslogs_appspot",
+    "datasetReference": {
+      "datasetId": "k8s_artifacts_gcslogs_appspot",
+      "projectId": "k8s-infra-ii-sandbox"
+    },
+    "location": "US"
+  },
+  {
+    "kind": "bigquery#dataset",
+    "id": "k8s-infra-ii-sandbox:kubernetes_public_logs",
+    "datasetReference": {
+      "datasetId": "kubernetes_public_logs",
+      "projectId": "k8s-infra-ii-sandbox"
+    },
+    "location": "US"
+  }
+]
diff --git a/...-ii-sandbox/services/bigquery/bigquery.datasets.k8s_artifacts_dataset_bb_test.access.json b/...-ii-sandbox/services/bigquery/bigquery.datasets.k8s_artifacts_dataset_bb_test.access.json
@@ -0,0 +1,18 @@
+[
+  {
+    "role": "WRITER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectOwners"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "bb@ii.coop"
+  },
+  {
+    "role": "READER",
+    "specialGroup": "projectReaders"
+  }
+]
diff --git a/...-ii-sandbox/services/bigquery/bigquery.datasets.k8s_artifacts_gcslogs_appspot.access.json b/...-ii-sandbox/services/bigquery/bigquery.datasets.k8s_artifacts_gcslogs_appspot.access.json
@@ -0,0 +1,18 @@
+[
+  {
+    "role": "WRITER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectOwners"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "bb@ii.coop"
+  },
+  {
+    "role": "READER",
+    "specialGroup": "projectReaders"
+  }
+]
diff --git a/...s-infra-ii-sandbox/services/bigquery/bigquery.datasets.kubernetes_public_logs.access.json b/...s-infra-ii-sandbox/services/bigquery/bigquery.datasets.kubernetes_public_logs.access.json
@@ -0,0 +1,18 @@
+[
+  {
+    "role": "WRITER",
+    "specialGroup": "projectWriters"
+  },
+  {
+    "role": "OWNER",
+    "specialGroup": "projectOwners"
+  },
+  {
+    "role": "OWNER",
+    "userByEmail": "caleb@ii.coop"
+  },
+  {
+    "role": "READER",
+    "specialGroup": "projectReaders"
+  }
+]