[terraform] remote terraform state (#12991)

infra/.gitignore

@@ -2,4 +2,3 @@
 /**/terraform.tfstate
 /**/terraform.tfstate*.backup
 /**/*.terraform.lock.hcl
-/**/*.tfvars

infra/azure/.gitignore

@@ -0,0 +1 @@
+*.tfvars

infra/gcp-broad/README.md

@@ -3,3 +3,86 @@ If you're a third-party trying to deploy Hail, look at `../gcp`.
 Hail team, this directory is an underestimate of our infrastructure. We are iteratively adding more
 infrastructure. Infrastructure may be missing because importing into terraform would require a
 destroy/create.
+
+---
+
+Changes from `../gcp`:
+
+Create a bucket in which to store terraform state. Use the same region in which you plan to create
+the k8s cluster.
+
+```
+PROJECT=YOUR GCP PROJECT HERE
+LOCATION=us-central1
+TERRAFORM_STATE_BUCKET=gs://terraform-state-$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 5)
+gsutil mb -l us-central1 $TERRAFORM_STATE_BUCKET
+gsutil -m uniformbucketlevelaccess set on $TERRAFORM_STATE_BUCKET
+```
+
+Create a key to encrypt terraform state.
+
+```
+gcloud kms keyrings create terraform-state-us-central1 \
+       --location $LOCATION
+gcloud kms keys create terraform-state-us-central1-key \
+       --location $LOCATION \
+	   --keyring terraform-state-us-central1 \
+	   --purpose encryption
+gcloud projects add-iam-policy-binding \
+       <project-id> \
+       --member='user:YOUR_EMAIL' \
+	   --role='roles/owner'
+gcloud kms keys list \
+       --location $LOCATION \
+	   --keyring terraform-state-us-central1
+```
+Store the Terraform key name in a variable for future use:
+```
+TERRAFORM_KEY_NAME=...
+```
+Finish the KMS setup:
+```
+gsutil kms authorize -p $PROJECT \
+       -k $TERRAFORM_KEY_NAME
+gcloud storage service-agent \
+       --project=$PROJECT \
+	   --authorize-cmek=$TERRAFORM_KEY_NAME
+gcloud storage buckets update \
+       $TERRAFORM_STATE_BUCKET \
+	   --default-encryption-key=$TERRAFORM_KEY_NAME
+```
+
+I found that I had to explicitly grant read permissions to my account even though it was an Owner:
+
+```
+YOUR_USER_EMAIL=...
+gcloud storage buckets add-iam-policy-binding \
+       --member user:$YOUR_USER_EMAIL \
+	   --role roles/storage.objectViewer \
+       $TERRAFORM_STATE_BUCKET
+gcloud storage buckets add-iam-policy-binding \
+       --member user:$YOUR_USER_EMAIL \
+	   --role roles/storage.objectCreator \
+       $TERRAFORM_STATE_BUCKET
+```
+
+Create `backend.hcl`:
+
+```
+cat >infra/gcp-broad/$GITHUB_ORGANIZATION/backend.hcl <<EOF
+bucket = $TERRAFORM_STATE_BUCKET
+kms_encryption_key = $TERRAFORM_KEY_NAME
+EOF
+```
+
+Initialize Terraform:
+
+```
+terraform init -backend-config=hail-is/backend.hcl -var-file=hail-is/global.tfvars
+```
+
+Then inspect the Terraform plan:
+
+```
+terraform plan -var-file=hail-is/global.tfvars -out=tfplan
+```

infra/gcp-broad/ci/main.tf

@@ -1,3 +1,12 @@
+terraform {
+  required_providers {
+    sops = {
+      source = "carlpett/sops"
+      version = "0.6.3"
+    }
+  }
+}
+
 resource "random_string" "hail_ci_bucket_suffix" {
   length = 5
 }
@@ -49,14 +58,24 @@ resource "kubernetes_secret" "ci_config" {
   }
 }
 
+data "sops_file" "zuliprc" {
+  count = fileexists("${var.github_organization}/zuliprc.enc") ? 1 : 0
+  source_file = "${var.github_organization}/zuliprc.enc"
+  input_type = "raw"
+}
+
+locals {
+    zuliprc = length(data.sops_file.zuliprc) == 1 ? data.sops_file.zuliprc[0] : null
+}
+
 resource "kubernetes_secret" "zulip_config" {
-  count = fileexists("~/.hail/.zuliprc") ? 1 : 0
+  count = local.zuliprc != null ? 1 : 0
   metadata {
     name = "zulip-config"
   }
 
   data = {
-    ".zuliprc" = fileexists("~/.hail/.zuliprc") ? file("~/.hail/.zuliprc") : ""
+    ".zuliprc" = local.zuliprc.raw
   }
 }
 

infra/gcp-broad/ci/variables.tf

@@ -35,3 +35,7 @@ variable "github_context" {
 variable "test_oauth2_callback_urls" {
   type = string
 }
+
+variable "github_organization" {
+  type = string
+}

infra/gcp-broad/hail-is/backend.hcl

@@ -0,0 +1,2 @@
+bucket = "terraform-state-50hs3"
+kms_encryption_key = "projects/hail-vdc/locations/us-central1/keyRings/terraform-state-us-central1/cryptoKeys/terraform-state-us-central1-key"

infra/gcp-broad/hail-is/ci_config.enc.json

@@ -0,0 +1,35 @@
+{
+	"deploy_steps": [],
+	"github_context": "ENC[AES256_GCM,data:JN5usSTHOg==,iv:CIA4ELD6Q/v06KbISOHeDCsnSjwWb51VpzrhN+3WKvs=,tag:EwSRVoerxqgxbx5+Q0G/Fg==,type:str]",
+	"storage_uri": "ENC[AES256_GCM,data:S9R83B6rVVTSozyBnfFpbeqJ,iv:85Y90Bg2jMGRt7lq3gaCQLxzQ+4pbRA+Hedw8/fGrak=,tag:ouZIcQAWBkgWll7GugW8ag==,type:str]",
+	"bucket_location": "ENC[AES256_GCM,data:AHTjUUbf6JamtYE=,iv:g9L8ceIQx0xUmE5xGHJJkwCrFilf165bYtjF2alVztU=,tag:QawH9d1WLm6SPLt7gBZnhQ==,type:str]",
+	"bucket_storage_class": "ENC[AES256_GCM,data:NX6xM/R9Bp8=,iv:WWskNj+PWD4UxWMsRyIqZroxIijNaadGJvFMJcH7NU8=,tag:ncV4dCcWEx+iF6RbsICKeQ==,type:str]",
+	"test_oauth2_callback_urls": "ENC[AES256_GCM,data:jObH1oS9+3OBoGu4VlQqj2OWlE2Fus3XuWYZ2pYWUq8pm3JYU7CK1KhWJfjioUh82f/67N2Y62JO+sI0+YBrpHk7cSWNCW4qrv/YSMRYr9x8VmcGg2kGIMXpcik9xdSidmpma4nWchXCARTuYATr0MHicDndxYkkQva0ImgWd3QrYRjjy/ZI1EGDyXIHSVTcSSqybErh/tM5vg8p0SnClrU7GSwFrMUV+80l7m1Qar+CIMd4gMYsBVTmdxEjeK5S+Z/tKQ434LduV8u21LrsQe4Q5DjgNRdgRNrn4llWfen9ie8+6onLjDfyfLr6/sh+50oGV6SFyEYPytKBToY2t1QnYrJaH/xcuGP9oPSDfvPdgdn7GzBUahcAEA72juIGN/RotnITws2iduAMNFjIFUEGKYVgFQWeHswmhzNhztIhKwlncq/RQzeuGVgB4prPTFqbT/7eT/57eHqFx84kedD7XEC5eCeqSdbCIcXMIzLAtwSFCHh/gL++luQ3jlriklw/bZ/OJVAISLWAgINKrmqbli7RW8gmDJ3Rlq2k20KgI2TLbAyUYCJm00jWTAGBy/jdrIQImrY2EIjdoBTnUFErcKIshkc7751R28lin22ybQCaXvmYfvy7Tuf0gp5pnn+U/znXLRBo/lW3kJadDilpkAcusdqlEtB6s/JcaBEhchVkfvpufJUXdyo5kMBjr2XSiWvu3xvMZbaMq59jdSwFhQu/tWCUdLL7bB8cXHshODihDvQGaEixx7NLffmzK3NO0aizBvOiPTyuBNRD+Q+RSSKKyi5AOqDSH3Umbhqe8RtMfUgYLZTsqozWBMopfsiECB+I8teapvpuJtI5+dvOmkIjrvcaUnZAVmRpcE7xF+wDUIZGETiscEYTv/7Y+G4ylLy6E5z4zk7ugiQjY79/H0xSm8mIYXyduDcFa4ZgWxYIZbghLgbi2FJj7J0+6SDm00eD1NfHkaozke7KvFlILW1VUOcrqqlxGzXlxQLoT27LxWQXSrwp+5SYvJHQfeSxT03ksHLHqoSa3lEMiYXHFh2mzwmtjK4wkkZpTUt0RJcfOnIy5SI1bkubnL0FU3ARbftYXL+aFWBdl86y7Hu+397srrZ4SZ2LNr3I33XzrHowUMxfvDLDAgSKxUCWuBm1EggaIQit7WMg9hDHxHc6QCWbupO0oFeGrvUbQoZZEqrLFc/cGbDGFZTURPrvTcE9Lrr2R7kzn3nKhHSAzCZrndhY5mCCOfR60cE04H857x1HhAJKxIsS3D1izDyQpoI//mbz+fKEEwDjWR5bnE1K8umy1CUMJYoPibzcVFpuBdTzvqwO4WeD8fiwdL6kFL788n/8+K4qERkVEip0PyhleAtvjxER4Xt/JSjNvikukfF07/GouPulnh8S7JeRGEgn0Lx/NeBLFbCQvdibsr/knCzDXbO5EoBd78AIkqBSD/2i1M4XEFsJNZOGpP9ZYNUUWdRrDyLKgqJkaEyTIyJ3mPiZcOo7htKyHol3,iv:BwWT2b31zTn1ELC9fZofIptqRDkJj2ybtAhwFQQtBKs=,tag:zkbT2TolS1XG7Ixa8lLwQw==,type:str]",
+	"watched_branches": [
+		[
+			"ENC[AES256_GCM,data:89ey6FPaU863AVhw/cXAJQk=,iv:mea/Zkiso71LCd+WMOyUzLxRhdTEvo+iCZCJEkOY9Hg=,tag:ay8YrryybQFeFGSwwRwcjA==,type:str]",
+			"ENC[AES256_GCM,data:UTQKiA==,iv:wRgansKSpx+LxP0qTXQtGjCaTjE6cjEDVFXFx7PYlZY=,tag:LSCQKiuXxRx8S/O04c57oQ==,type:bool]",
+			"ENC[AES256_GCM,data:4ER1XA==,iv:kBXFBG+9FV28yAPlrYHGrmVcoZKTryI6wt07fqrBr4k=,tag:6rdzwJ7e+14yScGqQGmmiA==,type:bool]"
+		]
+	],
+	"github_oauth_token": "ENC[AES256_GCM,data:vrcdC7SFoC92Gp9hzCzAsS19UsxEP1vfwY7kxHs4ngpjTvAmjDAYwg==,iv:uodttuaHtdOiw3jiwQPPhj6vkGOnEQsmw6XmrKnoCmg=,tag:gOeJGgOLj0gc6/C9X4et2w==,type:str]",
+	"github_user1_oauth_token": "ENC[AES256_GCM,data:4UpueIA+DJsqowuqT/CM6LWO23+AOCnla7F/En9uLm/KeTghKpxx3A==,iv:w70hLt2b1zTbyFZfuX9cjFtN2BJvvpM2/9qg1pGDlfc=,tag:uEprSZGsAyaEz5hZwKyB2w==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/hail-vdc/locations/global/keyRings/sops/cryptoKeys/sops-key",
+				"created_at": "2023-05-09T19:53:50Z",
+				"enc": "CiQA0ec+W0aC8bSdWP22np+CuMKieVOLkuzgXwQ9L3fYe8KuG9MSSQCe82Ad9tL4LzDEnBpALOvttYTDL8vIrBeS1IeFIimyNgfitkWSXcKWsB7hc08iI/8S/1pJRyUFpioMLI7aV0r0JZjeXajiofM="
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2023-05-09T19:53:50Z",
+		"mac": "ENC[AES256_GCM,data:ow71s2AziKKv6bmNPpddL7T87p8irYRzg+sLfYa2sHJ6OxSLpahT+Usjh70RE2Xmidycc8tvl+VFgFYUOFPocwEDReIbhM4B6gHbTThizgaV6tnIcSypzk+YUj/C3sl00668uOOtd5HAdnv1Ne1j4zOfuJGuXeVVZOB4zD+S25o=,iv:74+qhgXteCUeZKxh7GvTl7NPXgnoHPltwa8W0vBwG38=,tag:Yknyjubbr8ASIdZQ1RzPvA==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

infra/gcp-broad/hail-is/global.tfvars

@@ -0,0 +1,51 @@
+# organization_domain is a string that is the domain of the organization
+# E.g. "hail.is"
+organization_domain = "hail.is"
+
+# The GitHub organization hosting your Hail Batch repository, e.g. "hail-is".
+github_organization = "hail-is"
+
+# batch_gcp_regions is a JSON array of string, the names of the gcp
+# regions to schedule over in Batch. E.g. "[\"us-central1\"]"
+batch_gcp_regions = "[\"us-central1\"]"
+
+gcp_project = "hail-vdc"
+
+# This is the bucket location that spans the regions you're going to
+# schedule across in Batch.  If you are running on one region, it can
+# just be that region. E.g. "US"
+batch_logs_bucket_location = "us-central1"
+
+# The storage class for the batch logs bucket.  It should span the
+# batch regions and be compatible with the bucket location.
+batch_logs_bucket_storage_class = "STANDARD"
+
+# Similarly, bucket locations and storage classes are specified
+# for other services:
+hail_query_bucket_location = "us-central1"
+hail_query_bucket_storage_class = "STANDARD"
+hail_test_gcs_bucket_location = "us-central1"
+hail_test_gcs_bucket_storage_class = "STANDARD"
+
+# FIXME: what is this for
+gcp_region = "us-central1"
+
+# FIXME: what is this for
+gcp_zone = "us-central1-a"
+
+# FIXME: what is this for
+gcp_location = "us-central1"
+
+artifact_registry_location = "us"
+
+# FIXME: what is this for
+domain = "hail.is"
+
+# If set to true, pull the base ubuntu image from Artifact Registry.
+# Otherwise, assumes GCR.
+use_artifact_registry = true
+
+k8s_nonpreemptible_node_pool_name = "non-preemptible-pool-11"
+k8s_preemptible_node_pool_name = "preemptible-pool-8"
+
+default_subnet_ip_cidr_range = "10.128.0.0/20"

infra/gcp-broad/hail-is/zuliprc.enc

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:UFVTCuNj7+KYtY8yOTa8TOC5mnXScERGeI4Hof7OHMq31lUul2Om2FPBqv6B5RarzQjGCL1Ha2QM9hKL49LzXCpymEVlbTe5gI0Lxt95pmA1t34jPX3SNGYER2WVm7CaqJY7ONK8ag==,iv:XqkIgYP2SATlff/+dru6bU5Vch06pNMwtyS9BFlH3sg=,tag:3jwlRmOaBIhEVToNjSZ3Qw==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/hail-vdc/locations/global/keyRings/sops/cryptoKeys/sops-key",
+				"created_at": "2023-05-26T15:02:14Z",
+				"enc": "CiQA0ec+WwXRI2qJpgJhrIq19ChH432G8Ta7ViHw58FzKzvBMn8SSQCe82Adql2wUnT9vLQBn4k6Y3klU0ZCzz67jUCw9FKICCt4Z+P9FEkoISffqpompPEWS0p4STgXAj9yTQpVSmu9qJi4GLWeZ7w="
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2023-05-26T15:02:14Z",
+		"mac": "ENC[AES256_GCM,data:WJ2XxWZ4WOounHf90SIh4hbOZ0tM27XtkOjm1Spwt13nJtiBXnbhKdVETRLT1pcFqbRL9/bWixW9dVDwyyprbj8+mZ0qnyav7Tn2jd+zLb+O0YgoE+4syR0xFhquab4CahSsccZM73CptikunZ8iAhEoi76g/kEncCbk6pUQhV0=,iv:xRrufgYtPw4oXq+sDwWrQxSRMm2VaY8PQ0ZGbZHKoyU=,tag:1rLXU2imd5YHfJEtqSG6WQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

infra/gcp-broad/main.tf

@@ -13,6 +13,7 @@ terraform {
       version = "0.6.3"
     }
   }
+  backend "gcs" {}
 }
 
 variable "k8s_preemptible_node_pool_name" {
@@ -655,6 +656,10 @@ resource "google_storage_bucket" "hail_test_requester_pays_bucket" {
   uniform_bucket_level_access = true
   requester_pays = true
 
+  labels = {
+    "name" = "hail-test-requester-pays-fds32"
+  }
+
   timeouts {}
 }
 
@@ -748,4 +753,6 @@ module "ci" {
   ci_email = module.ci_gsa_secret.email
   github_context = local.ci_config.data["github_context"]
   test_oauth2_callback_urls = local.ci_config.data["test_oauth2_callback_urls"]
+
+  github_organization = var.github_organization
 }

infra/gcp/README.md

@@ -164,6 +164,24 @@ Instructions:
   # git commit and push as desired.
   ```
 
+- If you want Zulip integration for alerts from CI and Grafana, create a zuliprc file:
+
+  ```sh
+  cat /tmp/zuliprc <<EOF
+  [api]
+  key=SECRET_KEY_HERE
+  email=YOUR_BOT_EMAIL_HERE
+  site=YOUR_SITE_HERE
+  EOF
+  ```
+
+- Encrypt the zuliprc with SOPS:
+
+  ```sh
+  sops --encrypt --gcp-kms projects/<gcp-project-id>/locations/global/keyRings/sops/cryptoKeys/sops-key /tmp/zuliprc \
+       >$HAIL/infra/gcp/$GITHUB_ORGANIZATION/zuliprc.enc
+  ```
+
 - Install terraform.
 
 - Run `terraform init`.