From c83ffd7c612794facbb472b2da7c0467b885b435 Mon Sep 17 00:00:00 2001
From: Christoph Kleineweber <christoph.kleineweber@loodse.com>
Date: Mon, 18 Jan 2021 14:26:07 +0100
Subject: [PATCH 1/2] Add virtual IP address for API server in vSphere example

Set up keepalived on control plane nodes in vSphere Terraform example to
establish virtual IP address for Kubernetes API. API endpoint in
Terraform output is set to first control plane node if no IP address is
specified. The existing Gobetween load-balancer is removed.
---
 examples/terraform/vsphere/etc_gobetween.tpl  | 26 ------
 .../etc_keepalived_check_apiserver_sh.tpl     | 11 +++
 .../etc_keepalived_keepalived_conf.tpl        | 27 ++++++
 examples/terraform/vsphere/gobetween.sh       | 60 -------------
 examples/terraform/vsphere/keepalived.sh      | 33 +++++++
 examples/terraform/vsphere/main.tf            | 90 +++++++------------
 examples/terraform/vsphere/outputs.tf         |  2 +-
 examples/terraform/vsphere/variables.tf       | 14 +++
 8 files changed, 117 insertions(+), 146 deletions(-)
 delete mode 100644 examples/terraform/vsphere/etc_gobetween.tpl
 create mode 100644 examples/terraform/vsphere/etc_keepalived_check_apiserver_sh.tpl
 create mode 100644 examples/terraform/vsphere/etc_keepalived_keepalived_conf.tpl
 delete mode 100755 examples/terraform/vsphere/gobetween.sh
 create mode 100755 examples/terraform/vsphere/keepalived.sh

diff --git a/examples/terraform/vsphere/etc_gobetween.tpl b/examples/terraform/vsphere/etc_gobetween.tpl
deleted file mode 100644
index fdaae71a7..000000000
--- a/examples/terraform/vsphere/etc_gobetween.tpl
+++ /dev/null
@@ -1,26 +0,0 @@
-[api]
-enabled = false
-
-[servers.default]
-protocol = "tcp"
-bind = "0.0.0.0:6443"
-balance = "roundrobin"
-max_connections = 10000
-client_idle_timeout = "10m"
-backend_idle_timeout = "10m"
-backend_connection_timeout = "2s"
-
-[servers.default.discovery]
-kind = "static"
-static_list = [
-    %{ for target in lb_targets ~}
-    "${target}:6443",
-    %{ endfor ~}
-]
-
-[servers.default.healthcheck]
-kind = "ping"
-interval = "10s"
-timeout = "2s"
-fails = 2
-passes = 1
diff --git a/examples/terraform/vsphere/etc_keepalived_check_apiserver_sh.tpl b/examples/terraform/vsphere/etc_keepalived_check_apiserver_sh.tpl
new file mode 100644
index 000000000..b5f3bd7f7
--- /dev/null
+++ b/examples/terraform/vsphere/etc_keepalived_check_apiserver_sh.tpl
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+errorExit() {
+    echo "*** $*" 1>&2
+    exit 1
+}
+
+curl --silent --max-time 2 --insecure https://localhost:6443/healthz -o /dev/null || errorExit "Error GET https://localhost:6443/healthz"
+if ip addr | grep -q ${APISERVER_VIP}; then
+    curl --silent --max-time 2 --insecure https://${APISERVER_VIP}:6443/healthz -o /dev/null || errorExit "Error GET https://${APISERVER_VIP}:6443/healthz"
+fi
diff --git a/examples/terraform/vsphere/etc_keepalived_keepalived_conf.tpl b/examples/terraform/vsphere/etc_keepalived_keepalived_conf.tpl
new file mode 100644
index 000000000..165e39cd1
--- /dev/null
+++ b/examples/terraform/vsphere/etc_keepalived_keepalived_conf.tpl
@@ -0,0 +1,27 @@
+global_defs {
+    router_id LVS_DEVEL
+}
+vrrp_script check_apiserver {
+  script "/etc/keepalived/check_apiserver.sh"
+  interval 3
+  weight -2
+  fall 10
+  rise 2
+}
+
+vrrp_instance VI_1 {
+    state ${STATE}
+    interface ${INTERFACE}
+    virtual_router_id ${ROUTER_ID}
+    priority ${PRIORITY}
+    authentication {
+        auth_type PASS
+        auth_pass ${AUTH_PASS}
+    }
+    virtual_ipaddress {
+        ${APISERVER_VIP}
+    }
+    track_script {
+        check_apiserver
+    }
+}
diff --git a/examples/terraform/vsphere/gobetween.sh b/examples/terraform/vsphere/gobetween.sh
deleted file mode 100755
index 4a8da2204..000000000
--- a/examples/terraform/vsphere/gobetween.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2019 The KubeOne Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script is mostly used in CI
-# It installs dependencies and starts the tests
-
-set -euf -o pipefail
-
-GOBETWEEN_VERSION=0.7.0
-
-noop() { : "didn't detected package manager, noop"; }
-
-PKG_MANAGER="noop"
-
-[ "$(command -v yum)" ] && PKG_MANAGER=yum
-[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get
-
-sudo ${PKG_MANAGER} install tar -y
-
-mkdir -p /tmp/gobetween
-cd /tmp/gobetween
-curl -L -o gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz \
-    https://github.com/yyyar/gobetween/releases/download/${GOBETWEEN_VERSION}/gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
-tar xvf gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
-sudo mkdir -p /opt/bin
-sudo mv gobetween /opt/bin/gobetween
-sudo chown root:root /opt/bin/gobetween
-
-cat <<EOF | sudo tee /etc/systemd/system/gobetween.service
-[Unit]
-Description=Gobetween - modern LB for cloud era
-Documentation=https://github.com/yyyar/gobetween/wiki
-After=network.target remote-fs.target nss-lookup.target
-
-[Service]
-Type=simple
-ExecStart=/opt/bin/gobetween -c /etc/gobetween.toml
-PrivateTmp=true
-User=nobody
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-sudo systemctl daemon-reload
-sudo systemctl enable gobetween.service
-sudo systemctl start gobetween.service
diff --git a/examples/terraform/vsphere/keepalived.sh b/examples/terraform/vsphere/keepalived.sh
new file mode 100755
index 000000000..2b22fee2c
--- /dev/null
+++ b/examples/terraform/vsphere/keepalived.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The KubeOne Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is mostly used in CI
+# It installs dependencies and starts the tests
+
+set -euf -o pipefail
+
+noop() { : "didn't detected package manager, noop"; }
+
+PKG_MANAGER="noop"
+
+[ "$(command -v yum)" ] && PKG_MANAGER=yum
+[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get
+
+sudo ${PKG_MANAGER} update
+sudo ${PKG_MANAGER} install keepalived -y
+
+sudo systemctl enable keepalived.service
+sudo systemctl start keepalived.service
diff --git a/examples/terraform/vsphere/main.tf b/examples/terraform/vsphere/main.tf
index d785c8b8d..e84fa53b1 100644
--- a/examples/terraform/vsphere/main.tf
+++ b/examples/terraform/vsphere/main.tf
@@ -23,11 +23,6 @@ provider "vsphere" {
 
 locals {
   resource_pool_id = var.resource_pool_name == "" ? data.vsphere_compute_cluster.cluster.resource_pool_id : data.vsphere_resource_pool.pool[0].id
-
-  rendered_lb_config = templatefile("./etc_gobetween.tpl", {
-    lb_targets = vsphere_virtual_machine.control_plane.*.default_ip_address,
-  })
-
   hostnames = formatlist("${var.cluster_name}-cp-%d", [1, 2, 3])
 }
 
@@ -109,52 +104,6 @@ resource "vsphere_virtual_machine" "control_plane" {
       tags,
     ]
   }
-}
-
-resource "vsphere_virtual_machine" "lb" {
-  count            = 1
-  name             = "${var.cluster_name}-lb-${count.index + 1}"
-  resource_pool_id = local.resource_pool_id
-  folder           = var.folder_name
-  datastore_id     = data.vsphere_datastore.datastore.id
-  num_cpus         = 1
-  memory           = 1024
-  guest_id         = data.vsphere_virtual_machine.template.guest_id
-  scsi_type        = data.vsphere_virtual_machine.template.scsi_type
-
-  network_interface {
-    network_id   = data.vsphere_network.network.id
-    adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0]
-  }
-
-  disk {
-    label            = "disk0"
-    size             = var.disk_size
-    thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
-    eagerly_scrub    = data.vsphere_virtual_machine.template.disks[0].eagerly_scrub
-  }
-
-  cdrom {
-    client_device = true
-  }
-
-  clone {
-    template_uuid = data.vsphere_virtual_machine.template.id
-  }
-
-  vapp {
-    properties = {
-      hostname    = "${var.cluster_name}-lb-${count.index + 1}"
-      public-keys = file(var.ssh_public_key_file)
-    }
-  }
-
-  lifecycle {
-    ignore_changes = [
-      vapp[0].properties,
-      tags,
-    ]
-  }
 
   connection {
     type = "ssh"
@@ -163,30 +112,53 @@ resource "vsphere_virtual_machine" "lb" {
   }
 
   provisioner "remote-exec" {
-    script = "gobetween.sh"
+    script = "keepalived.sh"
   }
 }
 
-resource "null_resource" "lb_config" {
+resource "random_string" "keepalived_auth_pass" {
+  length  = 8
+  special = false
+}
+
+resource "null_resource" "keepalived_config" {
+  count = var.api_vip != "" ? 3 : 0
+
   triggers = {
     cluster_instance_ids = join(",", vsphere_virtual_machine.control_plane.*.id)
-    config               = local.rendered_lb_config
   }
 
   connection {
     user = var.ssh_username
-    host = vsphere_virtual_machine.lb[0].default_ip_address
+    host = vsphere_virtual_machine.control_plane[count.index].default_ip_address
+  }
+
+  provisioner "file" {
+    content = templatefile("./etc_keepalived_keepalived_conf.tpl", {
+      STATE         = count.index == 0 ? "MASTER" : "BACKUP",
+      APISERVER_VIP = var.api_vip,
+      INTERFACE     = var.vrrp_interface,
+      ROUTER_ID     = var.vrrp_router_id,
+      PRIORITY      = count.index == 0 ? "101" : "100",
+      AUTH_PASS     = random_string.keepalived_auth_pass.result
+    })
+    destination = "/tmp/keepalived.conf"
   }
 
   provisioner "file" {
-    content     = local.rendered_lb_config
-    destination = "/tmp/gobetween.toml"
+    content = templatefile("./etc_keepalived_check_apiserver_sh.tpl", {
+      APISERVER_VIP = var.api_vip
+    })
+    destination = "/tmp/check_apiserver.sh"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "sudo mv /tmp/gobetween.toml /etc/gobetween.toml",
-      "sudo systemctl restart gobetween",
+      "sudo mkdir -p /etc/keepalived",
+      "sudo mv /tmp/keepalived.conf /etc/keepalived/keepalived.conf",
+      "sudo mv /tmp/check_apiserver.sh /etc/keepalived/check_apiserver.sh",
+      "sudo chmod +x /etc/keepalived/check_apiserver.sh",
+      "sudo systemctl restart keepalived",
     ]
   }
 }
diff --git a/examples/terraform/vsphere/outputs.tf b/examples/terraform/vsphere/outputs.tf
index f578bb82f..fcf82bc59 100644
--- a/examples/terraform/vsphere/outputs.tf
+++ b/examples/terraform/vsphere/outputs.tf
@@ -18,7 +18,7 @@ output "kubeone_api" {
   description = "kube-apiserver LB endpoint"
 
   value = {
-    endpoint = vsphere_virtual_machine.lb[0].default_ip_address
+    endpoint = var.api_vip != "" ? var.api_vip : vsphere_virtual_machine.control_plane[0].default_ip_address
   }
 }
 
diff --git a/examples/terraform/vsphere/variables.tf b/examples/terraform/vsphere/variables.tf
index f8157757d..6f3f0724c 100644
--- a/examples/terraform/vsphere/variables.tf
+++ b/examples/terraform/vsphere/variables.tf
@@ -115,3 +115,17 @@ variable "worker_disk" {
   description = "disk size of each worker node in GB"
 }
 
+variable "api_vip" {
+  default     = ""
+  description = "virtual IP address for Kubernetes API"
+}
+
+variable "vrrp_interface" {
+  default = "ens192"
+  description = "network interface for API virtual IP"
+}
+
+variable "vrrp_router_id" {
+  default = 42
+  description = "vrrp router id for API virtual IP. Must be unique in used subnet"
+}

From dc546449db91ce0bd8289fa1c68c58aebf9ef31a Mon Sep 17 00:00:00 2001
From: Christoph Kleineweber <christoph.kleineweber@loodse.com>
Date: Mon, 18 Jan 2021 15:29:55 +0100
Subject: [PATCH 2/2] Update vSphere example README for virtual API IP

---
 examples/terraform/vsphere/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/terraform/vsphere/README.md b/examples/terraform/vsphere/README.md
index dd590183d..b9d568ca8 100644
--- a/examples/terraform/vsphere/README.md
+++ b/examples/terraform/vsphere/README.md
@@ -40,12 +40,15 @@ See the [Terraform loadbalancers in examples document][docs-tf-loadbalancer].
 | ssh\_username | SSH user, used only in output | string | `"root"` | no |
 | template\_name | template name | string | `"ubuntu-18.04"` | no |
 | worker\_os | OS to run on worker machines | string | `"ubuntu"` | no |
+| api_vip | Virtual IP address for Kubernetes API, established by keepalived" | string | `""` | no |
+| vrrp_interface | NIC to establish API IP address | string | `"ens192"` | no |
+| vrrp_router_id | Unique router id for VRRP protocol | int | 43 | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| kubeone\_api | kube-apiserver LB endpoint |
+| kubeone\_api | kube-apiserver endpoint. Either configured virtual IP or first node |
 | kubeone\_hosts | Control plane endpoints to SSH to |
 | kubeone\_workers | Workers definitions, that will be transformed into MachineDeployment object |