WIP: Use KVO to install cluster

Documentation/dev/node-bootstrap-flow.md

@@ -20,7 +20,9 @@ When a cluster node is being bootstrapped from scratch, it goes through several
 
 Additionally, only on one of the master nodes the following kubernetes bootstrapping happens:
 
-1. `bootkube.service` is started after `kubelet.service` start
+1. `kvo.service` is started after `kubelet.service` start
+1. `kvo.service` renders all bootstrap and self-hosted control plane assets
+1. `bootkube.service` is started after `kvo.service` start
 1. a static bootstrapping control-plane is deployed
 1. a fully self-hosted control-plane starts and takes over the previous one
 1. `bootkube.service` is completed with success
@@ -37,7 +39,8 @@ The following systemd units are deployed to a node by tectonic-installer and tak
 
 Additionally, only on one of the master nodes the following kubernetes bootstrapping happens:
 
-* `bootkube.service` deploys the initial bootstrapping control-plane. It is started only after `kubelet.service` _is started_. It is a oneshot unit and cannot crash, and it runs only during bootstrap
+* `kvo.service` renders all bootstrap and self-hosted control plane assets using the cluster config object that was pre-rendered by Terraform
+* `bootkube.service` deploys the initial bootstrapping control-plane. It is started only after `kvo.service` _is finished. It is a oneshot unit and cannot crash, and it runs only during bootstrap
 * `bootkube.path` waits for bootkube assets/scripts to exist on disk and triggers `bootkube.service`
 * `tectonic.service` deploys tectonic control-plane. It is started only after `bootkube.service` _has completed_.  It is a oneshot unit and cannot crash, and it runs only during bootstrap
 * `bootkube.path` waits for tectonic assets/scripts to exist on disk and triggers `tectonic.service`
@@ -80,6 +83,20 @@ On first boot, it is initially blocked by `k8s-node-bootstrap.service`.
 It crash-loop until the `kubelet.env` file exists.
 It is started on every boot.
 
+### `kvo.service`
+
+```
+Description=Generate resources for Bootkube
+ConditionPathExists=!/opt/tectonic/init_kvo.done
+Wants=kubelet.service
+After=kubelet.service
+Type=simple
+Restart=on-failure
+RemainAfterExit=true
+WorkingDirectory=/opt/tectonic
+```
+
+
 ### `bootkube.path` and `bootkube.service`
 
 ```

config.tf

@@ -55,7 +55,7 @@ variable "tectonic_container_images" {
   default = {
     addon_resizer                = "gcr.io/google_containers/addon-resizer:2.1"
     awscli                       = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
-    bootkube                     = "quay.io/coreos/bootkube:v0.6.2"
+    bootkube                     = "quay.io/coreos/bootkube-dev:6e1c2723123964ba91b4186483893672df8e3286"
     calico                       = "quay.io/calico/node:v2.4.1"
     calico_cni                   = "quay.io/calico/cni:v1.10.0"
     console                      = "quay.io/coreos/tectonic-console:v2.2.1"
@@ -73,7 +73,7 @@ variable "tectonic_container_images" {
     kubednsmasq                  = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
     kubedns_sidecar              = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
     kube_version                 = "quay.io/coreos/kube-version:0.1.0"
-    kube_version_operator        = "quay.io/coreos/kube-version-operator:v1.7.5-kvo.10"
+    kube_version_operator        = "quay.io/coreos/kube-version-operator-dev:4b53e2363d433b04bb22778bc984ec4c1149fe80"
     node_agent                   = "quay.io/coreos/node-agent:v1.7.5-kvo.3"
     pod_checkpointer             = "quay.io/coreos/pod-checkpointer:3517908b1a1837e78cfd041a0e51e61c7835d85f"
     stats_emitter                = "quay.io/coreos/tectonic-stats:6e882361357fe4b773adbf279cddf48cb50164c1"

modules/aws/master-asg/ignition.tf

@@ -14,6 +14,7 @@ data "ignition_config" "main" {
     var.ign_k8s_node_bootstrap_service_id,
     data.ignition_systemd_unit.init_assets.id,
     var.ign_bootkube_service_id,
+    var.ign_kvo_service_id,
     var.ign_tectonic_service_id,
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,

modules/aws/master-asg/variables.tf

@@ -130,6 +130,11 @@ variable "ign_bootkube_service_id" {
   description = "The ID of the bootkube systemd service unit"
 }
 
+variable "ign_kvo_service_id" {
+  type        = "string"
+  description = "The ID of the kvo systemd service unit"
+}
+
 variable "ign_bootkube_path_unit_id" {
   type = "string"
 }

modules/bootkube/assets.tf

@@ -50,11 +50,13 @@ resource "template_dir" "bootkube" {
   destination_dir = "./generated/manifests"
 
   vars {
-    hyperkube_image        = "${var.container_images["hyperkube"]}"
-    pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}"
-    kubedns_image          = "${var.container_images["kubedns"]}"
-    kubednsmasq_image      = "${var.container_images["kubednsmasq"]}"
-    kubedns_sidecar_image  = "${var.container_images["kubedns_sidecar"]}"
+    kube_version_operator_image = "${var.container_images["kube_version_operator"]}"
+    pull_secret                 = "${base64encode(file(var.pull_secret_path))}"
+    hyperkube_image             = "${var.container_images["hyperkube"]}"
+    pod_checkpointer_image      = "${var.container_images["pod_checkpointer"]}"
+    kubedns_image               = "${var.container_images["kubedns"]}"
+    kubednsmasq_image           = "${var.container_images["kubednsmasq"]}"
+    kubedns_sidecar_image       = "${var.container_images["kubedns_sidecar"]}"
 
     # Choose the etcd endpoints to use.
     # 1. If experimental mode is enabled (self-hosted etcd), then use
@@ -185,17 +187,66 @@ resource "local_file" "bootkube_sh" {
   filename = "./generated/bootkube.sh"
 }
 
+resource "local_file" "kvo_config" {
+  content  = "${data.template_file.kvo_config.rendered}"
+  filename = "./generated/kvo-config.yaml"
+}
+
+resource "local_file" "pull_secret" {
+  content  = "${file(var.pull_secret_path)}"
+  filename = "./generated/config.json"
+}
+
 # bootkube.service (available as output variable)
 data "template_file" "bootkube_service" {
   template = "${file("${path.module}/resources/bootkube.service")}"
 }
 
+# kvo.service (available as output variable)
+data "template_file" "kvo_service" {
+  template = "${file("${path.module}/resources/kvo.service")}"
+  vars {
+    kube_version_operator_image = "${var.container_images["kube_version_operator"]}"
+    kubernetes_version          = "${var.versions["kubernetes"]}"
+  }
+}
+
+data "template_file" "kvo_config" {
+  template = "${file("${path.module}/resources/kvo-config.yaml")}"
+  vars {
+    advertise_address      = "${var.advertise_address}"
+    cloud_provider_profile = "${var.cloud_provider != "" ? "${var.cloud_provider}" : "metal"}"
+    cloud_config_path      = "${var.cloud_config_path}"
+    cluster_cidr           = "${var.cluster_cidr}"
+    master_count           = "${var.master_count}"
+    oidc_issuer_url        = "${var.oidc_issuer_url}"
+    oidc_client_id         = "${var.oidc_client_id}"
+    oidc_username_claim    = "${var.oidc_username_claim}"
+    oidc_groups_claim      = "${var.oidc_groups_claim}"
+    service_cidr           = "${var.service_cidr}"
+
+    etcd_servers = "${
+      var.experimental_enabled
+        ? format("https://%s:2379", cidrhost(var.service_cidr, 15))
+        : var.etcd_ca_cert_pem == ""
+          ? join(",", formatlist("http://%s:2379", var.etcd_endpoints))
+          : join(",", formatlist("https://%s:2379", var.etcd_endpoints))
+      }"
+  }
+}
+
 data "ignition_systemd_unit" "bootkube_service" {
   name    = "bootkube.service"
   enable  = false
   content = "${data.template_file.bootkube_service.rendered}"
 }
 
+data "ignition_systemd_unit" "kvo_service" {
+  name    = "kvo.service"
+  enable  = true
+  content = "${data.template_file.kvo_service.rendered}"
+}
+
 # bootkube.path (available as output variable)
 data "template_file" "bootkube_path_unit" {
   template = "${file("${path.module}/resources/bootkube.path")}"

modules/bootkube/outputs.tf

@@ -19,7 +19,9 @@ output "id" {
   value = "${sha1("
   ${local_file.kubeconfig.id}
   ${local_file.bootkube_sh.id}
-  ${template_dir.bootkube.id} ${template_dir.bootkube_bootstrap.id}
+  ${local_file.pull_secret.id}
+  ${local_file.kvo_config.id}
+  ${template_dir.bootkube.id}
   ${join(" ",
     template_dir.experimental.*.id,
     template_dir.bootstrap_experimental.*.id,
@@ -40,6 +42,14 @@ output "systemd_service_id" {
   value = "${data.ignition_systemd_unit.bootkube_service.id}"
 }
 
+output "kvo_config_id" {
+  value = "${data.template_file.kvo_config.id}"
+}
+
+output "kvo_systemd_service_id" {
+  value = "${data.ignition_systemd_unit.kvo_service.id}"
+}
+
 output "systemd_path_unit_rendered" {
   value = "${data.template_file.bootkube_path_unit.rendered}"
 }

modules/bootkube/resources/bootkube.service

@@ -1,8 +1,8 @@
 [Unit]
 Description=Bootstrap a Kubernetes cluster
 ConditionPathExists=!/opt/tectonic/init_bootkube.done
-Wants=kubelet.service
-After=kubelet.service
+Wants=kvo.service
+After=kvo.service
 
 [Service]
 Type=oneshot
@@ -16,4 +16,4 @@ ExecStart=/usr/bin/bash /opt/tectonic/bootkube.sh
 ExecStartPost=/bin/touch /opt/tectonic/init_bootkube.done
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

modules/bootkube/resources/kvo-config.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: KubeVersionOperatorConfig
+authConfig:
+  oidc_client_id: ${oidc_client_id}
+  oidc_issuer_url: ${oidc_issuer_url}
+  oidc_groups_claim: ${oidc_groups_claim}
+  oidc_username_claim: ${oidc_username_claim}
+cloudProviderConfig:
+  cloud_config_path: ${cloud_config_path}
+  cloud_provider_profile: ${cloud_provider_profile}
+networkConfig:
+  advertise_address: ${advertise_address}
+  cluster_cidr: ${cluster_cidr}
+  etcd_servers: ${etcd_servers}
+  service_cidr: ${service_cidr}
+initialConfig:
+  initial_master_count: ${master_count}

modules/bootkube/resources/kvo.service

@@ -0,0 +1,31 @@
+[Unit]
+Description=Generate resources for Bootkube
+ConditionPathExists=!/opt/tectonic/init_kvo.done
+Wants=kubelet.service
+After=kubelet.service
+
+[Service]
+Type=simple
+Restart=on-failure
+RemainAfterExit=true
+WorkingDirectory=/opt/tectonic
+
+User=root
+Group=root
+
+ExecStartPre=/usr/bin/docker \
+            --config=/opt/tectonic \
+            run --rm \
+            -v /opt/tectonic:/opt/tectonic \
+            ${kube_version_operator_image} \
+            /kube-version-operator \
+            --upgrade-spec=/upgrade-spec.yaml \
+            --version=${kubernetes_version} \
+            --cluster-config=/opt/tectonic/kvo-config.yaml \
+            --outdir=/opt/tectonic \
+            render
+
+ExecStartPost=/bin/touch /opt/tectonic/init_kvo.done
+
+[Install]
+WantedBy=multi-user.target

modules/tectonic/resources/manifests/updater/app_versions/app-version-kubernetes.yaml → modules/bootkube/resources/manifests/03-kubernetes-app-version.yaml

@@ -9,5 +9,4 @@ spec:
   desiredVersion: ${kubernetes_version}
   paused: false
 status:
-  currentVersion: ${kubernetes_version}
   paused: false