From 94df14e78d25e7b95bc869fca97769d1cb8ab573 Mon Sep 17 00:00:00 2001
From: Nolan Reisbeck <nolan.reisbeck@gmail.com>
Date: Tue, 17 Oct 2017 22:10:17 -0700
Subject: [PATCH] master alignment

---
 modules/digitalocean/etcd/etcd.tf             |  14 ++
 modules/digitalocean/etcd/ignition.tf         |  12 --
 modules/digitalocean/etcd/variables.tf        |  33 ++---
 modules/digitalocean/master/ignition.tf       |  40 ++++++
 modules/digitalocean/master/master.tf         |   4 +-
 .../master/resources/init-assets.sh           |   9 ++
 .../resources/services/init-assets.service    |  19 +++
 .../resources/services/kubelet-env.service    |  13 ++
 .../master/resources/services/kubelet.service |  43 +++++++
 .../digitalocean/master/variables-ignition.tf |   1 +
 modules/digitalocean/master/variables.tf      |  19 ++-
 modules/digitalocean/worker/ignition.tf       |  13 ++
 .../digitalocean/worker/variables-ignition.tf |   1 +
 modules/digitalocean/worker/variables.tf      |  25 ++--
 modules/digitalocean/worker/worker.tf         |   2 +-
 modules/ignition/assets.tf                    |  14 --
 modules/ignition/outputs.tf                   |   4 -
 modules/ignition/variables.tf                 |   5 -
 platforms/digitalocean/main.tf                |  17 +--
 .../resources/bootstrap-first-master.sh       |   2 +-
 platforms/digitalocean/tectonic.tf            | 121 ++++++++++--------
 platforms/digitalocean/variables.tf           |   6 +-
 22 files changed, 268 insertions(+), 149 deletions(-)
 create mode 100644 modules/digitalocean/etcd/etcd.tf
 create mode 100644 modules/digitalocean/master/ignition.tf
 create mode 100644 modules/digitalocean/master/resources/init-assets.sh
 create mode 100644 modules/digitalocean/master/resources/services/init-assets.service
 create mode 100644 modules/digitalocean/master/resources/services/kubelet-env.service
 create mode 100644 modules/digitalocean/master/resources/services/kubelet.service
 create mode 120000 modules/digitalocean/master/variables-ignition.tf
 create mode 100644 modules/digitalocean/worker/ignition.tf
 create mode 120000 modules/digitalocean/worker/variables-ignition.tf

diff --git a/modules/digitalocean/etcd/etcd.tf b/modules/digitalocean/etcd/etcd.tf
new file mode 100644
index 0000000000..f32cd2432f
--- /dev/null
+++ b/modules/digitalocean/etcd/etcd.tf
@@ -0,0 +1,14 @@
+resource "digitalocean_droplet" "etcd_node" {
+  count     = "${var.droplet_count}"
+  name      = "${var.cluster_name}-etcd-${count.index}"
+  image     = "coreos-${var.container_linux_channel}"
+  region    = "${var.droplet_region}"
+  size      = "${var.droplet_size}"
+  ssh_keys  = ["${var.ssh_keys}"]
+  tags      = ["${var.extra_tags}"]
+  user_data = "${data.ignition_config.etcd.*.rendered[count.index]}"
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
diff --git a/modules/digitalocean/etcd/ignition.tf b/modules/digitalocean/etcd/ignition.tf
index a0c4f14b5f..b852ab8ec2 100644
--- a/modules/digitalocean/etcd/ignition.tf
+++ b/modules/digitalocean/etcd/ignition.tf
@@ -5,8 +5,6 @@ data "ignition_config" "etcd" {
     "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}",
     "${data.ignition_systemd_unit.etcd3.*.id[count.index]}",
     "${data.ignition_systemd_unit.etcd_unzip_tls.id}",
-    "${module.swap.service_id}",
-    "${module.sshguard.service_id}",
   ]
 
   files = [
@@ -121,13 +119,3 @@ EOF
     },
   ]
 }
-
-module "swap" {
-  source = "../../swap"
-
-  swap_size = "${var.swap_size}"
-}
-
-module "sshguard" {
-  source = "../../sshguard"
-}
diff --git a/modules/digitalocean/etcd/variables.tf b/modules/digitalocean/etcd/variables.tf
index 17050cfe88..937a7bbed8 100644
--- a/modules/digitalocean/etcd/variables.tf
+++ b/modules/digitalocean/etcd/variables.tf
@@ -1,4 +1,4 @@
-variable "cluster_linux_channel" {
+variable "base_domain" {
   type = "string"
 }
 
@@ -6,14 +6,26 @@ variable "cluster_name" {
   type = "string"
 }
 
-variable "base_domain" {
+variable "container_image" {
+  type = "string"
+}
+
+variable "container_linux_channel" {
   type = "string"
 }
 
+variable "container_linux_version" {
+   type = "string"
+}
+
 variable "droplet_count" {
   default = "3"
 }
 
+variable "droplet_region" {
+  type = "string"
+}
+
 variable "droplet_size" {
   type = "string"
 }
@@ -23,27 +35,10 @@ variable "extra_tags" {
   default = []
 }
 
-variable "container_image" {
-  type = "string"
-}
-
 variable "ssh_keys" {
   type = "list"
 }
 
-variable "droplet_region" {
-  type = "string"
-}
-
-# variable "droplet_image" {
-#   type = "string"
-# }
-
-variable "swap_size" {
-  type        = "string"
-  description = "Amount of swap memory to enable"
-}
-
 variable "tls_enabled" {
   default = false
 }
diff --git a/modules/digitalocean/master/ignition.tf b/modules/digitalocean/master/ignition.tf
new file mode 100644
index 0000000000..d2d2ffeb49
--- /dev/null
+++ b/modules/digitalocean/master/ignition.tf
@@ -0,0 +1,40 @@
+data "ignition_config" "main" {
+  files = [
+    "${data.ignition_file.init_assets.id}",
+    "${var.ign_installer_kubelet_env_id}",
+    "${var.ign_max_user_watches_id}",
+    "${data.ignition_file.init_assets.id}",
+  ]
+
+  systemd = ["${compact(list(
+    var.ign_docker_dropin_id,
+    var.ign_locksmithd_service_id,
+    var.ign_kubelet_service_id,
+    var.ign_k8s_node_bootstrap_service_id,
+    var.ign_init_assets_service_id,
+    var.ign_bootkube_service_id,
+    var.ign_tectonic_service_id,
+    var.ign_bootkube_path_unit_id,
+    var.ign_tectonic_path_unit_id
+   ))}"]
+}
+
+data "template_file" "init_assets" {
+  template = "${file("${path.module}/resources/init-assets.sh")}"
+
+  vars {
+    cluster_name        = "${var.cluster_name}"
+    kubelet_image_url   = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}"
+    kubelet_image_tag   = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}"
+  }
+}
+
+data "ignition_file" "init_assets" {
+  filesystem = "root"
+  path       = "/opt/init-assets.sh"
+  mode       = 0755
+
+  content {
+    content = "${data.template_file.init_assets.rendered}"
+  }
+}
diff --git a/modules/digitalocean/master/master.tf b/modules/digitalocean/master/master.tf
index 326ac18c1c..5a1af77e58 100644
--- a/modules/digitalocean/master/master.tf
+++ b/modules/digitalocean/master/master.tf
@@ -1,7 +1,7 @@
 resource "digitalocean_droplet" "master_node" {
   count = "${var.master_count}"
   name      = "${var.cluster_name}-master-${count.index}"
-  image     = "coreos-${module.container_linux_channel}"
+  image     = "coreos-${var.container_linux_channel}"
   region    = "${var.droplet_region}"
   size      = "${var.droplet_size}"
   ssh_keys  = ["${var.ssh_keys}"]
@@ -47,7 +47,7 @@ resource "digitalocean_loadbalancer" "console" {
 }
 
 resource "digitalocean_domain" "cluster" {
-  name       = "cluster.${var.cluster_name}.${var.base_domain}"
+  name       = "${var.cluster_name}.${var.base_domain}"
   ip_address = "${digitalocean_droplet.master_node.*.ipv4_address[0]}"
 }
 
diff --git a/modules/digitalocean/master/resources/init-assets.sh b/modules/digitalocean/master/resources/init-assets.sh
new file mode 100644
index 0000000000..08c8a890f7
--- /dev/null
+++ b/modules/digitalocean/master/resources/init-assets.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+# Populate the kubelet.env file
+mkdir -p /etc/kubernetes
+echo "KUBELET_IMAGE_URL=${kubelet_image_url}" > /etc/kubernetes/kubelet.env
+echo "KUBELET_IMAGE_TAG=${kubelet_image_tag}" >> /etc/kubernetes/kubelet.env
+
+exit 0
diff --git a/modules/digitalocean/master/resources/services/init-assets.service b/modules/digitalocean/master/resources/services/init-assets.service
new file mode 100644
index 0000000000..244c2feb71
--- /dev/null
+++ b/modules/digitalocean/master/resources/services/init-assets.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=Initialize Tectonic Assets
+ConditionPathExists=!/opt/tectonic/init-assets.done
+Before=bootkube.service k8s-node-bootstrap-service.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+WorkingDirectory=/opt/tectonic
+
+User=root
+Group=root
+
+ExecStart=/usr/bin/bash /opt/tectonic/init-assets.sh
+ExecStartPost=/bin/touch /opt/tectonic/init-assets.done
+
+[Install]
+WantedBy=multi-user.target
+RequiredBy=bootkube.service k8s-node-bootstrap-service.service
diff --git a/modules/digitalocean/master/resources/services/kubelet-env.service b/modules/digitalocean/master/resources/services/kubelet-env.service
new file mode 100644
index 0000000000..63bdac8985
--- /dev/null
+++ b/modules/digitalocean/master/resources/services/kubelet-env.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Determine the Kubelet Image Version
+ConditionPathExists=!/etc/kubernetes/kubelet.env
+
+[Service]
+ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes
+ExecStartPre=/usr/bin/bash -c "docker run --rm -v /etc/kubernetes:/etc/kubernetes ${kube_version_image_url}:${kube_version_image_tag} --kubeconfig=/etc/kubernetes/kubeconfig > /etc/kubernetes/kube.version"
+ExecStart=/usr/bin/bash -c "echo KUBELET_IMAGE_URL=${kubelet_image_url} > /etc/kubernetes/kubelet.env; echo KUBELET_IMAGE_TAG=$(tr '+' '_' < /etc/kubernetes/kube.version) >> /etc/kubernetes/kubelet.env; rm /etc/kubernetes/kube.version"
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
diff --git a/modules/digitalocean/master/resources/services/kubelet.service b/modules/digitalocean/master/resources/services/kubelet.service
new file mode 100644
index 0000000000..fd1e0696fa
--- /dev/null
+++ b/modules/digitalocean/master/resources/services/kubelet.service
@@ -0,0 +1,43 @@
+[Unit]
+Description=Kubelet via Hyperkube ACI
+
+[Service]
+EnvironmentFile=/etc/kubernetes/kubelet.env
+Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
+  --volume=resolv,kind=host,source=/etc/resolv.conf \
+  --mount volume=resolv,target=/etc/resolv.conf \
+  --volume var-lib-cni,kind=host,source=/var/lib/cni \
+  --mount volume=var-lib-cni,target=/var/lib/cni \
+  --volume var-log,kind=host,source=/var/log \
+  --mount volume=var-log,target=/var/log"
+
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests \
+  /srv/kubernetes/manifests /etc/kubernetes/checkpoint-secrets \
+  /etc/kubernetes/cni/net.d /var/lib/cni
+ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
+
+ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
+  --cni-conf-dir=/etc/kubernetes/cni/net.d \
+  --network-plugin=cni \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --exit-on-lock-contention \
+  --pod-manifest-path=/etc/kubernetes/manifests \
+  --allow-privileged \
+  --node-labels=${node_label} \
+  ${node_taints_param} \
+  --minimum-container-ttl-duration=6m0s \
+  --cluster-dns=${cluster_dns_ip} \
+  --cluster-domain=cluster.local \
+  --client-ca-file=/etc/kubernetes/ca.crt \
+  --anonymous-auth=false \
+  --cloud-provider=
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
diff --git a/modules/digitalocean/master/variables-ignition.tf b/modules/digitalocean/master/variables-ignition.tf
new file mode 120000
index 0000000000..e22a48c662
--- /dev/null
+++ b/modules/digitalocean/master/variables-ignition.tf
@@ -0,0 +1 @@
+../../ignition/outputs.import
\ No newline at end of file
diff --git a/modules/digitalocean/master/variables.tf b/modules/digitalocean/master/variables.tf
index e51d42d3d7..e17299339e 100644
--- a/modules/digitalocean/master/variables.tf
+++ b/modules/digitalocean/master/variables.tf
@@ -2,10 +2,6 @@ variable "base_domain" {
   type = "string"
 }
 
-variable "container_linux_channel" {
-  type = "string"
-}
-
 variable "cluster_name" {
   type = "string"
 }
@@ -15,14 +11,22 @@ variable "container_images" {
   type        = "map"
 }
 
-variable "droplet_size" {
+variable "container_linux_channel" {
   type = "string"
 }
 
+variable "container_linux_version" {
+   type = "string"
+}
+
 variable "droplet_region" {
   type = "string"
 }
 
+variable "droplet_size" {
+  type = "string"
+}
+
 variable "extra_tags" {
   type    = "list"
   default = []
@@ -34,7 +38,6 @@ variable "image_re" {
 }
 
 variable "master_count" {
-  #type    = "string"
   default = 1
 }
 
@@ -42,10 +45,6 @@ variable "ssh_keys" {
   type = "list"
 }
 
-# variable "user_data" {
-#    type = "string"
-# }
-
 variable "ign_init_assets_service_id" {
   type = "string"
 }
diff --git a/modules/digitalocean/worker/ignition.tf b/modules/digitalocean/worker/ignition.tf
new file mode 100644
index 0000000000..3114c72e89
--- /dev/null
+++ b/modules/digitalocean/worker/ignition.tf
@@ -0,0 +1,13 @@
+data "ignition_config" "main" {
+  files = [
+    "${var.ign_installer_kubelet_env_id}",
+    "${var.ign_max_user_watches_id}",
+  ]
+
+  systemd = [
+    "${var.ign_docker_dropin_id}",
+    "${var.ign_k8s_node_bootstrap_service_id}",
+    "${var.ign_kubelet_service_id}",
+    "${var.ign_locksmithd_service_id}",
+  ]
+}
diff --git a/modules/digitalocean/worker/variables-ignition.tf b/modules/digitalocean/worker/variables-ignition.tf
new file mode 120000
index 0000000000..e22a48c662
--- /dev/null
+++ b/modules/digitalocean/worker/variables-ignition.tf
@@ -0,0 +1 @@
+../../ignition/outputs.import
\ No newline at end of file
diff --git a/modules/digitalocean/worker/variables.tf b/modules/digitalocean/worker/variables.tf
index 4019580ed0..f9066b9e58 100644
--- a/modules/digitalocean/worker/variables.tf
+++ b/modules/digitalocean/worker/variables.tf
@@ -1,5 +1,4 @@
-variable "droplet_count" {
-variable "container_linux_channel" {
+variable "base_domain" {
   type = "string"
 }
 
@@ -7,30 +6,30 @@ variable "cluster_name" {
   type = "string"
 }
 
-variable "droplet_image" {
+variable "container_linux_channel" {
   type = "string"
 }
 
-variable "droplet_region" {
+variable "container_linux_version" {
+   type = "string"
+}
+
+variable "droplet_count" {
   type = "string"
 }
 
-variable "droplet_size" {
+variable "droplet_region" {
   type = "string"
 }
 
-variable "ssh_keys" {
-  type = "list"
+variable "droplet_size" {
+  type = "string"
 }
 
 variable "extra_tags" {
   type = "list"
 }
 
-# variable "user_data" {
-#   type = "string"
-# }
-
-variable "base_domain" {
-  type = "string"
+variable "ssh_keys" {
+  type = "list"
 }
diff --git a/modules/digitalocean/worker/worker.tf b/modules/digitalocean/worker/worker.tf
index 1a382ec96a..0784a5dd8a 100644
--- a/modules/digitalocean/worker/worker.tf
+++ b/modules/digitalocean/worker/worker.tf
@@ -1,7 +1,7 @@
 resource "digitalocean_droplet" "worker_node" {
   count     = "${var.droplet_count}"
   name      = "${var.cluster_name}-worker-${count.index}"
-  image     = "coreos-${module.container_linux_channel}"
+  image     = "coreos-${var.container_linux_channel}"
   region    = "${var.droplet_region}"
   size      = "${var.droplet_size}"
   ssh_keys  = ["${var.ssh_keys}"]
diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf
index b09878b215..edcdffb858 100644
--- a/modules/ignition/assets.tf
+++ b/modules/ignition/assets.tf
@@ -172,17 +172,3 @@ data "ignition_systemd_unit" "coreos_metadata" {
     },
   ]
 }
-
-data "template_file" "swap" {
-  template = "${file("${path.module}/resources/services/swap.service")}"
-
-  vars {
-    swap_size = "${var.swap_size}"
-  }
-}
-
-data "ignition_systemd_unit" "swap" {
-  name    = "swap.service"
-  enable  = "${var.swap_size != "" ? true : false}"
-  content = "${data.template_file.swap.rendered}"
-}
diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf
index 34062fb119..4690ad7be6 100644
--- a/modules/ignition/outputs.tf
+++ b/modules/ignition/outputs.tf
@@ -93,7 +93,3 @@ output "coreos_metadata_dropin_id" {
 output "coreos_metadata_dropin_rendered" {
   value = "${data.template_file.coreos_metadata.rendered}"
 }
-
-output "service_id" {
-  value = "${data.ignition_systemd_unit.swap.id}"
-}
diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf
index 2561b4c8b9..a377e75efc 100644
--- a/modules/ignition/variables.tf
+++ b/modules/ignition/variables.tf
@@ -111,8 +111,3 @@ variable "metadata_provider" {
 variable "use_metadata" {
   default = true
 }
-
-variable "swap_size" {
-  type = "string"
-  description = "Amount of swap memory to enable"
-}
diff --git a/platforms/digitalocean/main.tf b/platforms/digitalocean/main.tf
index 7edacd613c..9c2413ae8c 100644
--- a/platforms/digitalocean/main.tf
+++ b/platforms/digitalocean/main.tf
@@ -15,8 +15,8 @@ module "etcd" {
   source = "../../modules/digitalocean/etcd"
 
   base_domain             = "${var.tectonic_base_domain}"
-  container_linux_channel = "${module.tectonic_container_linux_channel}"
-  container_linux_version = "${module.tectonic_container_linux_version}"
+  container_linux_channel = "${var.tectonic_container_linux_channel}"
+  container_linux_version = "${module.container_linux.version}"
   cluster_name            = "${var.tectonic_cluster_name}"
   container_image         = "${var.tectonic_container_images["etcd"]}"
   droplet_count           = "${var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : 3}"
@@ -25,7 +25,6 @@ module "etcd" {
 
   extra_tags              = "${var.tectonic_do_extra_tags}"
   ssh_keys                = "${var.tectonic_do_ssh_keys}"
-  swap_size               = "${var.tectonic_do_etcd_swap}"
   tls_enabled             = "${var.tectonic_etcd_tls_enabled}"
   tls_zip                 = "${module.etcd_certs.etcd_tls_zip}"
 }
@@ -46,7 +45,6 @@ module "ignition_masters" {
   kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
   kubelet_node_label        = "node-role.kubernetes.io/master"
   kubelet_node_taints       = "node-role.kubernetes.io/master=:NoSchedule"
-  swap_size                 = "${var.tectonic_do_master_swap}"
   tectonic_vanilla_k8s      = "${var.tectonic_vanilla_k8s}"
 }
 
@@ -54,8 +52,8 @@ module "masters" {
   source = "../../modules/digitalocean/master"
 
   base_domain             = "${var.tectonic_base_domain}"
-  container_linux_channel = "${module.tectonic_container_linux_channel}"
-  container_linux_version = "${module.tectonic_container_linux_version}"
+  container_linux_channel = "${var.tectonic_container_linux_channel}"
+  container_linux_version = "${module.container_linux.version}"
   cluster_name            = "${var.tectonic_cluster_name}"
   container_images        = "${var.tectonic_container_images}"
   droplet_size            = "${var.tectonic_do_master_droplet_size}"
@@ -63,8 +61,6 @@ module "masters" {
   extra_tags              = "${var.tectonic_do_extra_tags}"
   ssh_keys                = "${var.tectonic_do_ssh_keys}"
 
-  #user_data = "${module.ignition.ignition_config.main.rendered}"
-
   ign_bootkube_path_unit_id         = "${module.bootkube.systemd_path_unit_id}"
   ign_bootkube_service_id           = "${module.bootkube.systemd_service_id}"
   ign_docker_dropin_id              = "${module.ignition_masters.docker_dropin_id}"
@@ -90,15 +86,14 @@ module "ignition_workers" {
   kubelet_node_label   = "node-role.kubernetes.io/node"
   kubelet_node_taints  = ""
   tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
-  swap_size            = "${var.tectonic_do_worker_swap}"
 }
 
 module "workers" {
   source = "../../modules/digitalocean/worker"
 
   base_domain             = "${var.tectonic_base_domain}"
-  container_linux_channel = "${module.tectonic_container_linux_channel}"
-  container_linux_version = "${module.tectonic_container_linux_version}"
+  container_linux_channel = "${var.tectonic_container_linux_channel}"
+  container_linux_version = "${module.container_linux.version}"
   cluster_name            = "${var.tectonic_cluster_name}"
   droplet_count           = "${var.tectonic_worker_count}"
   droplet_size            = "${var.tectonic_do_worker_droplet_size}"
diff --git a/platforms/digitalocean/resources/bootstrap-first-master.sh b/platforms/digitalocean/resources/bootstrap-first-master.sh
index 3ca431e581..a653d35922 100644
--- a/platforms/digitalocean/resources/bootstrap-first-master.sh
+++ b/platforms/digitalocean/resources/bootstrap-first-master.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 sudo unzip -o -d /opt/tectonic/ $HOME/tectonic.zip && \
-rm $HOME/tectonic.zip && \
+#rm $HOME/tectonic.zip && \
 sudo systemctl enable bootkube && \
 sudo systemctl start bootkube
 
diff --git a/platforms/digitalocean/tectonic.tf b/platforms/digitalocean/tectonic.tf
index f1950bd941..e2bf95165f 100644
--- a/platforms/digitalocean/tectonic.tf
+++ b/platforms/digitalocean/tectonic.tf
@@ -43,11 +43,9 @@ module "identity_certs" {
 
 module "bootkube" {
   source         = "../../modules/bootkube"
-  cloud_provider = "digitalocean"
 
+  cloud_provider = "digitalocean"
   cluster_name = "${var.tectonic_cluster_name}"
-
-  ## TODO Add private endpoints
   kube_apiserver_url = "https://${module.masters.cluster_fqdn}:443"
   oidc_issuer_url    = "https://${module.masters.console_fqdn}:443/identity"
 
@@ -136,17 +134,12 @@ module "flannel_vxlan" {
   cluster_cidr      = "${var.tectonic_cluster_cidr}"
   enabled           = "${var.tectonic_networking == "flannel"}"
   container_images = "${var.tectonic_container_images}"
-  #flannel_image     = "${var.tectonic_container_images["flannel"]}"
-  #flannel_cni_image = "${var.tectonic_container_images["flannel_cni"]}"
 }
 
 module "calico" {
   source = "../../modules/net/calico"
 
-  #kube_apiserver_url = "https://${module.masters.cluster_fqdn}:443"
   container_images   = "${var.tectonic_container_images}"
-  #calico_image       = "${var.tectonic_container_images["calico"]}"
-  #calico_cni_image   = "${var.tectonic_container_images["calico_cni"]}"
   cluster_cidr       = "${var.tectonic_cluster_cidr}"
   enabled            = "${var.tectonic_networking == "calico"}"
 }
@@ -178,7 +171,7 @@ data "archive_file" "assets" {
 
 # Copy kubeconfig to master nodes
 resource "null_resource" "master_nodes" {
-  count = 1
+  count = "${var.tectonic_master_count}"
 
   # Re-provision on changes to masters
   triggers {
@@ -203,22 +196,6 @@ resource "null_resource" "master_nodes" {
       "sudo mv $HOME/kubeconfig /etc/kubernetes/",
     ]
   }
-}
-
-# Copy assets to first master node
-resource "null_resource" "first_master" {
-  # Re-provision on changes to first master node
-  triggers {
-    node_address = "${module.masters.first_node_address}"
-  }
-
-  connection {
-    type        = "ssh"
-    host        = "${module.masters.first_node_address}"
-    user        = "core"
-    private_key = "${file("${var.tectonic_do_ssh_key_path}")}"
-    timeout     = "1m"
-  }
 
   provisioner "file" {
     source      = "${data.archive_file.assets.output_path}"
@@ -234,36 +211,72 @@ resource "null_resource" "first_master" {
     inline = [
       "chmod +x $HOME/bootstrap-first-master.sh",
       "$HOME/bootstrap-first-master.sh ${var.tectonic_vanilla_k8s ? "" : "--enable-tectonic"}",
-      "rm $HOME/bootstrap-first-master.sh",
-    ]
-  }
-}
-
-# Copy kubeconfig to worker nodes
-resource "null_resource" "worker_nodes" {
-  count = "${var.tectonic_worker_count}"
 
-  # Re-provision on changes to workers
-  triggers {
-    node_address = "${element(module.workers.node_addresses, count.index)}"
-  }
-
-  connection {
-    type        = "ssh"
-    host        = "${element(module.workers.node_addresses, count.index)}"
-    user        = "core"
-    private_key = "${file("${var.tectonic_do_ssh_key_path}")}"
-    timeout     = "1m"
-  }
-
-  provisioner "file" {
-    content     = "${module.bootkube.kubeconfig}"
-    destination = "$HOME/kubeconfig"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "sudo mv $HOME/kubeconfig /etc/kubernetes/",
     ]
+    # move up one
+    #"rm $HOME/bootstrap-first-master.sh",
   }
 }
+
+# Copy assets to first master node
+# resource "null_resource" "first_master" {
+#   # Re-provision on changes to first master node
+#   triggers {
+#     node_address = "${module.masters.first_node_address}"
+#   }
+#
+#   connection {
+#     type        = "ssh"
+#     host        = "${module.masters.first_node_address}"
+#     user        = "core"
+#     private_key = "${file("${var.tectonic_do_ssh_key_path}")}"
+#     timeout     = "1m"
+#   }
+#
+#   provisioner "file" {
+#     source      = "${data.archive_file.assets.output_path}"
+#     destination = "$HOME/tectonic.zip"
+#   }
+#
+#   provisioner "file" {
+#     source      = "${path.root}/resources/bootstrap-first-master.sh"
+#     destination = "$HOME/bootstrap-first-master.sh"
+#   }
+#
+#   provisioner "remote-exec" {
+#     inline = [
+#       "chmod +x $HOME/bootstrap-first-master.sh",
+#       "$HOME/bootstrap-first-master.sh ${var.tectonic_vanilla_k8s ? "" : "--enable-tectonic"}",
+#       "rm $HOME/bootstrap-first-master.sh",
+#     ]
+#   }
+# }
+#
+# # Copy kubeconfig to worker nodes
+# resource "null_resource" "worker_nodes" {
+#   count = "${var.tectonic_worker_count}"
+#
+#   # Re-provision on changes to workers
+#   triggers {
+#     node_address = "${element(module.workers.node_addresses, count.index)}"
+#   }
+#
+#   connection {
+#     type        = "ssh"
+#     host        = "${element(module.workers.node_addresses, count.index)}"
+#     user        = "core"
+#     private_key = "${file("${var.tectonic_do_ssh_key_path}")}"
+#     timeout     = "1m"
+#   }
+#
+#   provisioner "file" {
+#     content     = "${module.bootkube.kubeconfig}"
+#     destination = "$HOME/kubeconfig"
+#   }
+#
+#   provisioner "remote-exec" {
+#     inline = [
+#       "sudo mv $HOME/kubeconfig /etc/kubernetes/",
+#     ]
+#   }
+# }
diff --git a/platforms/digitalocean/variables.tf b/platforms/digitalocean/variables.tf
index 09d62e344c..3e970ffacd 100644
--- a/platforms/digitalocean/variables.tf
+++ b/platforms/digitalocean/variables.tf
@@ -47,7 +47,7 @@ variable "tectonic_do_extra_tags" {
 }
 
 variable "tectonic_do_private_networking" {
-  type "string"
-  description = "Enable private networking for this cluster. Default: true"
-  default = "true"
+  type        =  "string"
+  description = "Enable private networking for this cluster. (Default) true"
+  default     = "true"
 }