Add virtual IP address for API server in vSphere example

examples/terraform/vsphere/README.md

@@ -40,12 +40,15 @@ See the [Terraform loadbalancers in examples document][docs-tf-loadbalancer].
 | ssh\_username | SSH user, used only in output | string | `"root"` | no |
 | template\_name | template name | string | `"ubuntu-18.04"` | no |
 | worker\_os | OS to run on worker machines | string | `"ubuntu"` | no |
+| api_vip | Virtual IP address for Kubernetes API, established by keepalived" | string | `""` | no |
+| vrrp_interface | NIC to establish API IP address | string | `"ens192"` | no |
+| vrrp_router_id | Unique router id for VRRP protocol | int | 43 | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| kubeone\_api | kube-apiserver LB endpoint |
+| kubeone\_api | kube-apiserver endpoint. Either configured virtual IP or first node |
 | kubeone\_hosts | Control plane endpoints to SSH to |
 | kubeone\_workers | Workers definitions, that will be transformed into MachineDeployment object |
 

examples/terraform/vsphere/etc_keepalived_check_apiserver_sh.tpl

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+errorExit() {
+    echo "*** $*" 1>&2
+    exit 1
+}
+
+curl --silent --max-time 2 --insecure https://localhost:6443/healthz -o /dev/null || errorExit "Error GET https://localhost:6443/healthz"
+if ip addr | grep -q ${APISERVER_VIP}; then
+    curl --silent --max-time 2 --insecure https://${APISERVER_VIP}:6443/healthz -o /dev/null || errorExit "Error GET https://${APISERVER_VIP}:6443/healthz"
+fi

examples/terraform/vsphere/etc_keepalived_keepalived_conf.tpl

@@ -0,0 +1,27 @@
+global_defs {
+    router_id LVS_DEVEL
+}
+vrrp_script check_apiserver {
+  script "/etc/keepalived/check_apiserver.sh"
+  interval 3
+  weight -2
+  fall 10
+  rise 2
+}
+
+vrrp_instance VI_1 {
+    state ${STATE}
+    interface ${INTERFACE}
+    virtual_router_id ${ROUTER_ID}
+    priority ${PRIORITY}
+    authentication {
+        auth_type PASS
+        auth_pass ${AUTH_PASS}
+    }
+    virtual_ipaddress {
+        ${APISERVER_VIP}
+    }
+    track_script {
+        check_apiserver
+    }
+}

examples/terraform/vsphere/keepalived.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The KubeOne Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is mostly used in CI
+# It installs dependencies and starts the tests
+
+set -euf -o pipefail
+
+noop() { : "didn't detected package manager, noop"; }
+
+PKG_MANAGER="noop"
+
+[ "$(command -v yum)" ] && PKG_MANAGER=yum
+[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get
+
+sudo ${PKG_MANAGER} update
+sudo ${PKG_MANAGER} install keepalived -y
+
+sudo systemctl enable keepalived.service
+sudo systemctl start keepalived.service

examples/terraform/vsphere/main.tf

@@ -23,11 +23,6 @@ provider "vsphere" {
 
 locals {
   resource_pool_id = var.resource_pool_name == "" ? data.vsphere_compute_cluster.cluster.resource_pool_id : data.vsphere_resource_pool.pool[0].id
-
-  rendered_lb_config = templatefile("./etc_gobetween.tpl", {
-    lb_targets = vsphere_virtual_machine.control_plane.*.default_ip_address,
-  })
-
   hostnames = formatlist("${var.cluster_name}-cp-%d", [1, 2, 3])
 }
 
@@ -109,52 +104,6 @@ resource "vsphere_virtual_machine" "control_plane" {
       tags,
     ]
   }
-}
-
-resource "vsphere_virtual_machine" "lb" {
-  count            = 1
-  name             = "${var.cluster_name}-lb-${count.index + 1}"
-  resource_pool_id = local.resource_pool_id
-  folder           = var.folder_name
-  datastore_id     = data.vsphere_datastore.datastore.id
-  num_cpus         = 1
-  memory           = 1024
-  guest_id         = data.vsphere_virtual_machine.template.guest_id
-  scsi_type        = data.vsphere_virtual_machine.template.scsi_type
-
-  network_interface {
-    network_id   = data.vsphere_network.network.id
-    adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0]
-  }
-
-  disk {
-    label            = "disk0"
-    size             = var.disk_size
-    thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
-    eagerly_scrub    = data.vsphere_virtual_machine.template.disks[0].eagerly_scrub
-  }
-
-  cdrom {
-    client_device = true
-  }
-
-  clone {
-    template_uuid = data.vsphere_virtual_machine.template.id
-  }
-
-  vapp {
-    properties = {
-      hostname    = "${var.cluster_name}-lb-${count.index + 1}"
-      public-keys = file(var.ssh_public_key_file)
-    }
-  }
-
-  lifecycle {
-    ignore_changes = [
-      vapp[0].properties,
-      tags,
-    ]
-  }
 
   connection {
     type = "ssh"
@@ -163,30 +112,53 @@ resource "vsphere_virtual_machine" "lb" {
   }
 
   provisioner "remote-exec" {
-    script = "gobetween.sh"
+    script = "keepalived.sh"
   }
 }
 
-resource "null_resource" "lb_config" {
+resource "random_string" "keepalived_auth_pass" {
+  length  = 8
+  special = false
+}
+
+resource "null_resource" "keepalived_config" {
+  count = var.api_vip != "" ? 3 : 0
+
   triggers = {
     cluster_instance_ids = join(",", vsphere_virtual_machine.control_plane.*.id)
-    config               = local.rendered_lb_config
   }
 
   connection {
     user = var.ssh_username
-    host = vsphere_virtual_machine.lb[0].default_ip_address
+    host = vsphere_virtual_machine.control_plane[count.index].default_ip_address
+  }
+
+  provisioner "file" {
+    content = templatefile("./etc_keepalived_keepalived_conf.tpl", {
+      STATE         = count.index == 0 ? "MASTER" : "BACKUP",
+      APISERVER_VIP = var.api_vip,
+      INTERFACE     = var.vrrp_interface,
+      ROUTER_ID     = var.vrrp_router_id,
+      PRIORITY      = count.index == 0 ? "101" : "100",
+      AUTH_PASS     = random_string.keepalived_auth_pass.result
+    })
+    destination = "/tmp/keepalived.conf"
   }
 
   provisioner "file" {
-    content     = local.rendered_lb_config
-    destination = "/tmp/gobetween.toml"
+    content = templatefile("./etc_keepalived_check_apiserver_sh.tpl", {
+      APISERVER_VIP = var.api_vip
+    })
+    destination = "/tmp/check_apiserver.sh"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "sudo mv /tmp/gobetween.toml /etc/gobetween.toml",
-      "sudo systemctl restart gobetween",
+      "sudo mkdir -p /etc/keepalived",
+      "sudo mv /tmp/keepalived.conf /etc/keepalived/keepalived.conf",
+      "sudo mv /tmp/check_apiserver.sh /etc/keepalived/check_apiserver.sh",
+      "sudo chmod +x /etc/keepalived/check_apiserver.sh",
+      "sudo systemctl restart keepalived",
     ]
   }
 }

examples/terraform/vsphere/outputs.tf

@@ -18,7 +18,7 @@ output "kubeone_api" {
   description = "kube-apiserver LB endpoint"
 
   value = {
-    endpoint = vsphere_virtual_machine.lb[0].default_ip_address
+    endpoint = var.api_vip != "" ? var.api_vip : vsphere_virtual_machine.control_plane[0].default_ip_address
   }
 }
 

examples/terraform/vsphere/variables.tf

@@ -115,3 +115,17 @@ variable "worker_disk" {
   description = "disk size of each worker node in GB"
 }
 
+variable "api_vip" {
+  default     = ""
+  description = "virtual IP address for Kubernetes API"
+}
+
+variable "vrrp_interface" {
+  default = "ens192"
+  description = "network interface for API virtual IP"
+}
+
+variable "vrrp_router_id" {
+  default = 42
+  description = "vrrp router id for API virtual IP. Must be unique in used subnet"
+}