Use Availability Set for masters and workers

Instances in a Scale Set are not supported by the kubernetes
cloud-provider integration.

This closes coreos#93

modules/azure/master-as/master.tf

@@ -0,0 +1,93 @@
+# TODO:
+# Create global network tf file
+# Add azurerm_route_table
+# Add azurerm_network_security_group
+# Add azurerm_availability_set
+
+# Generate unique storage name
+resource "random_id" "tectonic_master_storage_name" {
+  byte_length = 4
+}
+
+resource "azurerm_storage_account" "tectonic_master" {
+  name                = "${random_id.tectonic_master_storage_name.hex}"
+  resource_group_name = "${var.resource_group_name}"
+  location            = "${var.location}"
+  account_type        = "${var.storage_account_type}"
+
+  tags {
+    environment = "staging"
+  }
+}
+
+resource "azurerm_storage_container" "tectonic_master" {
+  name                  = "${var.cluster_name}-vhd-master"
+  resource_group_name   = "${var.resource_group_name}"
+  storage_account_name  = "${azurerm_storage_account.tectonic_master.name}"
+  container_access_type = "private"
+}
+
+resource "azurerm_availability_set" "tectonic_masters" {
+  name                = "${var.cluster_name}-masters"
+  location            = "${var.location}"
+  resource_group_name = "${var.resource_group_name}"
+}
+
+resource "azurerm_network_interface" "tectonic_master" {
+  count               = "${var.master_count}"
+  name                = "${var.cluster_name}-master${count.index}"
+  location            = "${var.location}"
+  resource_group_name = "${var.resource_group_name}"
+
+  ip_configuration {
+    private_ip_address_allocation           = "dynamic"
+    name                                    = "${var.cluster_name}-MasterIPConfiguration"
+    subnet_id                               = "${var.subnet}"
+    load_balancer_backend_address_pools_ids = ["${azurerm_lb_backend_address_pool.api-lb.id}"]
+  }
+}
+
+resource "azurerm_virtual_machine" "tectonic_master" {
+  count                 = "${var.master_count}"
+  name                  = "${var.cluster_name}-master${count.index}"
+  location              = "${var.location}"
+  resource_group_name   = "${var.resource_group_name}"
+  network_interface_ids = ["${element(azurerm_network_interface.tectonic_master.*.id, count.index)}"]
+  vm_size               = "${var.vm_size}"
+
+  storage_image_reference {
+    publisher = "CoreOS"
+    offer     = "CoreOS"
+    sku       = "Stable"
+    version   = "latest"
+  }
+
+  storage_os_disk {
+    name          = "master-osdisk"
+    caching       = "ReadWrite"
+    create_option = "FromImage"
+    os_type       = "linux"
+    vhd_uri       = "${azurerm_storage_account.tectonic_master.primary_blob_endpoint}${azurerm_storage_container.tectonic_master.name}/${var.cluster_name}-master${count.index}.vhd"
+  }
+
+  os_profile {
+    computer_name  = "${var.cluster_name}-master${count.index}"
+    admin_username = "core"
+    admin_password = ""
+
+    custom_data = "${base64encode("${data.ignition_config.master.*.rendered[count.index]}")}"
+  }
+
+  os_profile_linux_config {
+    disable_password_authentication = true
+
+    ssh_keys {
+      path     = "/home/core/.ssh/authorized_keys"
+      key_data = "${file(var.public_ssh_key)}"
+    }
+  }
+
+  tags {
+    environment = "staging"
+  }
+}

modules/azure/master-ss/api-lb.tf

@@ -0,0 +1,60 @@
+resource "azurerm_public_ip" "tectonic_api_ip" {
+  name                         = "tectonic_api_ip"
+  location                     = "${var.location}"
+  resource_group_name          = "${var.resource_group_name}"
+  public_ip_address_allocation = "static"
+  domain_name_label            = "${var.cluster_name}-k8s"
+
+  tags {
+    environment = "staging"
+  }
+}
+
+resource "azurerm_lb_rule" "api-lb" {
+  name                    = "api-lb-rule-443-443"
+  resource_group_name     = "${var.resource_group_name}"
+  loadbalancer_id         = "${azurerm_lb.tectonic_lb.id}"
+  backend_address_pool_id = "${azurerm_lb_backend_address_pool.api-lb.id}"
+  probe_id                = "${azurerm_lb_probe.api-lb.id}"
+
+  protocol                       = "tcp"
+  frontend_port                  = 443
+  backend_port                   = 443
+  frontend_ip_configuration_name = "api"
+}
+
+resource "azurerm_lb_probe" "api-lb" {
+  name                = "api-lb-probe-443-up"
+  loadbalancer_id     = "${azurerm_lb.tectonic_lb.id}"
+  resource_group_name = "${var.resource_group_name}"
+  protocol            = "tcp"
+  port                = 443
+}
+
+resource "azurerm_lb_backend_address_pool" "api-lb" {
+  name                = "api-lb-pool"
+  resource_group_name = "${var.resource_group_name}"
+  loadbalancer_id     = "${azurerm_lb.tectonic_lb.id}"
+}
+
+resource "azurerm_lb_rule" "ssh-lb" {
+  name                    = "ssh-lb"
+  resource_group_name     = "${var.resource_group_name}"
+  loadbalancer_id         = "${azurerm_lb.tectonic_lb.id}"
+  backend_address_pool_id = "${azurerm_lb_backend_address_pool.api-lb.id}"
+  probe_id                = "${azurerm_lb_probe.ssh-lb.id}"
+  load_distribution       = "SourceIP"
+
+  protocol                       = "tcp"
+  frontend_port                  = 22
+  backend_port                   = 22
+  frontend_ip_configuration_name = "api"
+}
+
+resource "azurerm_lb_probe" "ssh-lb" {
+  name                = "ssh-lb-22-up"
+  loadbalancer_id     = "${azurerm_lb.tectonic_lb.id}"
+  resource_group_name = "${var.resource_group_name}"
+  protocol            = "tcp"
+  port                = 22
+}

modules/azure/master-ss/console-lb.tf

@@ -0,0 +1,45 @@
+resource "azurerm_public_ip" "tectonic_console_ip" {
+  name                         = "tectonic_console_ip"
+  location                     = "${var.location}"
+  resource_group_name          = "${var.resource_group_name}"
+  public_ip_address_allocation = "static"
+  domain_name_label            = "${var.cluster_name}"
+
+  tags {
+    environment = "staging"
+  }
+}
+
+resource "azurerm_lb_rule" "console-lb-https" {
+  name                    = "console-lb-rule-443-32000"
+  resource_group_name     = "${var.resource_group_name}"
+  loadbalancer_id         = "${azurerm_lb.tectonic_lb.id}"
+  backend_address_pool_id = "${azurerm_lb_backend_address_pool.api-lb.id}"
+  probe_id                = "${azurerm_lb_probe.console-lb.id}"
+
+  protocol                       = "tcp"
+  frontend_port                  = 443
+  backend_port                   = 32000
+  frontend_ip_configuration_name = "console"
+}
+
+resource "azurerm_lb_rule" "console-lb-identity" {
+  name                    = "console-lb-rule-80-32001"
+  resource_group_name     = "${var.resource_group_name}"
+  loadbalancer_id         = "${azurerm_lb.tectonic_lb.id}"
+  backend_address_pool_id = "${azurerm_lb_backend_address_pool.api-lb.id}"
+  probe_id                = "${azurerm_lb_probe.console-lb.id}"
+
+  protocol                       = "tcp"
+  frontend_port                  = 80
+  backend_port                   = 32001
+  frontend_ip_configuration_name = "console"
+}
+
+resource "azurerm_lb_probe" "console-lb" {
+  name                = "console-lb-probe-443-up"
+  loadbalancer_id     = "${azurerm_lb.tectonic_lb.id}"
+  resource_group_name = "${var.resource_group_name}"
+  protocol            = "tcp"
+  port                = 32000
+}

modules/azure/master-ss/ignition-master.tf

@@ -0,0 +1,105 @@
+data "ignition_config" "master" {
+  files = [
+    "${data.ignition_file.kubeconfig.id}",
+    "${data.ignition_file.kubelet-env.id}",
+    "${data.ignition_file.max-user-watches.id}",
+  ]
+
+  systemd = [
+    "${data.ignition_systemd_unit.docker.id}",
+    "${data.ignition_systemd_unit.locksmithd.id}",
+    "${data.ignition_systemd_unit.kubelet-master.id}",
+    "${data.ignition_systemd_unit.tectonic.id}",
+    "${data.ignition_systemd_unit.bootkube.id}",
+  ]
+
+  users = [
+    "${data.ignition_user.core.id}",
+  ]
+}
+
+data "ignition_user" "core" {
+  name = "core"
+
+  ssh_authorized_keys = [
+    "${file(var.public_ssh_key)}",
+  ]
+}
+
+data "ignition_systemd_unit" "docker" {
+  name   = "docker.service"
+  enable = true
+
+  dropin = [
+    {
+      name    = "10-dockeropts.conf"
+      content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n"
+    },
+  ]
+}
+
+data "ignition_systemd_unit" "locksmithd" {
+  name = "locksmithd.service"
+  mask = true
+}
+
+data "template_file" "kubelet-master" {
+  template = "${file("${path.module}/resources/master-kubelet.service")}"
+
+  vars {
+    node_label        = "${var.kubelet_node_label}"
+    node_taints_param = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}"
+    cloud_provider    = "${var.cloud_provider}"
+    cluster_dns       = "${var.tectonic_kube_dns_service_ip}"
+  }
+}
+
+data "ignition_systemd_unit" "kubelet-master" {
+  name    = "kubelet.service"
+  enable  = true
+  content = "${data.template_file.kubelet-master.rendered}"
+}
+
+data "ignition_file" "kubeconfig" {
+  filesystem = "root"
+  path       = "/etc/kubernetes/kubeconfig"
+  mode       = 0644
+
+  content {
+    content = "${var.kubeconfig_content}"
+  }
+}
+
+data "ignition_file" "kubelet-env" {
+  filesystem = "root"
+  path       = "/etc/kubernetes/kubelet.env"
+  mode       = 0644
+
+  content {
+    content = <<EOF
+KUBELET_IMAGE_URL="${var.kube_image_url}"
+KUBELET_IMAGE_TAG="${var.kube_image_tag}"
+EOF
+  }
+}
+
+data "ignition_file" "max-user-watches" {
+  filesystem = "root"
+  path       = "/etc/sysctl.d/max-user-watches.conf"
+  mode       = 0644
+
+  content {
+    content = "fs.inotify.max_user_watches=16184"
+  }
+}
+
+data "ignition_systemd_unit" "bootkube" {
+  name    = "bootkube.service"
+  content = "${var.bootkube_service}"
+}
+
+data "ignition_systemd_unit" "tectonic" {
+  name    = "tectonic.service"
+  enable  = "${var.tectonic_service_disabled == 0 ? true : false}"
+  content = "${var.tectonic_service}"
+}

modules/azure/master-ss/lb.tf

@@ -0,0 +1,17 @@
+resource "azurerm_lb" "tectonic_lb" {
+  name                = "api-lb"
+  location            = "${var.location}"
+  resource_group_name = "${var.resource_group_name}"
+
+  frontend_ip_configuration {
+    name                          = "api"
+    public_ip_address_id          = "${azurerm_public_ip.tectonic_api_ip.id}"
+    private_ip_address_allocation = "dynamic"
+  }
+
+  frontend_ip_configuration {
+    name                          = "console"
+    public_ip_address_id          = "${azurerm_public_ip.tectonic_console_ip.id}"
+    private_ip_address_allocation = "dynamic"
+  }
+}

modules/azure/master-ss/output.tf

@@ -0,0 +1,23 @@
+output "ip_address" {
+  value = ["${azurerm_public_ip.tectonic_api_ip.ip_address}"]
+}
+
+output "console_ip_address" {
+  value = "${azurerm_public_ip.tectonic_console_ip.ip_address}"
+}
+
+output "ingress_external_fqdn" {
+  value = "${var.use_custom_fqdn ? "${azurerm_public_ip.tectonic_console_ip.domain_name_label}.${var.base_domain}" : azurerm_public_ip.tectonic_console_ip.fqdn}"
+}
+
+output "ingress_internal_fqdn" {
+  value = "${var.use_custom_fqdn ? "${azurerm_public_ip.tectonic_console_ip.domain_name_label}.${var.base_domain}" : azurerm_public_ip.tectonic_console_ip.fqdn}"
+}
+
+output "api_external_fqdn" {
+  value = "${var.use_custom_fqdn ?  "${azurerm_public_ip.tectonic_api_ip.domain_name_label}.${var.base_domain}" : azurerm_public_ip.tectonic_api_ip.fqdn}"
+}
+
+output "api_internal_fqdn" {
+  value = "${var.use_custom_fqdn ?  "${azurerm_public_ip.tectonic_api_ip.domain_name_label}.${var.base_domain}" : azurerm_public_ip.tectonic_api_ip.fqdn}"
+}

modules/azure/master-ss/resources/master-kubelet.service

@@ -0,0 +1,42 @@
+[Unit]
+Description=Kubelet via Hyperkube ACI
+
+[Service]
+Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
+  --volume=resolv,kind=host,source=/etc/resolv.conf \
+  --mount volume=resolv,target=/etc/resolv.conf \
+  --volume var-lib-cni,kind=host,source=/var/lib/cni \
+  --mount volume=var-lib-cni,target=/var/lib/cni \
+  --volume var-log,kind=host,source=/var/log \
+  --mount volume=var-log,target=/var/log"
+EnvironmentFile=/etc/kubernetes/kubelet.env
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+ExecStartPre=/bin/mkdir -p /var/lib/cni
+ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
+ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
+  --cni-conf-dir=/etc/kubernetes/cni/net.d \
+  --network-plugin=cni \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --exit-on-lock-contention \
+  --pod-manifest-path=/etc/kubernetes/manifests \
+  --allow-privileged \
+  --node-labels=${node_label} \
+  ${node_taints_param} \
+  --minimum-container-ttl-duration=6m0s \
+  --cluster_dns=${cluster_dns} \
+  --cluster_domain=cluster.local \
+  --client-ca-file=/etc/kubernetes/ca.crt \
+  --anonymous-auth=false \
+  --cloud-provider="${cloud_provider}"
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target