Merge pull request #422 from kinvolk/johananl/packet-cloudflare-dns

Packet: Add Cloudflare DNS support

assets/lokomotive-kubernetes/dns/cloudflare/main.tf

@@ -0,0 +1,37 @@
+data "cloudflare_zones" "selected" {
+  filter {
+    name   = var.dns_zone
+    status = "active"
+    paused = false
+  }
+}
+
+resource "cloudflare_record" "apiserver_public" {
+  count = length(var.controllers_public_ipv4)
+
+  zone_id = lookup(data.cloudflare_zones.selected.zones[0], "id")
+  name    = format("%s.%s.", var.cluster_name, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  value   = var.controllers_public_ipv4[count.index]
+}
+
+resource "cloudflare_record" "apiserver_private" {
+  count = length(var.controllers_private_ipv4)
+
+  zone_id = lookup(data.cloudflare_zones.selected.zones[0], "id")
+  name    = format("%s-private.%s.", var.cluster_name, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  value   = var.controllers_private_ipv4[count.index]
+}
+
+resource "cloudflare_record" "etcd" {
+  count = length(var.controllers_private_ipv4)
+
+  zone_id = lookup(data.cloudflare_zones.selected.zones[0], "id")
+  name    = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  value   = var.controllers_private_ipv4[count.index]
+}

assets/lokomotive-kubernetes/dns/cloudflare/require.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">= 0.12.0"
+
+  required_providers {
+    cloudflare = "~> 2.0"
+  }
+}

assets/lokomotive-kubernetes/dns/cloudflare/variables.tf

@@ -0,0 +1 @@
+../shared-variables.tf

assets/lokomotive-kubernetes/dns/manual/main.tf

@@ -0,0 +1,38 @@
+locals {
+  api_external_fqdn = format("%s.%s.", var.cluster_name, var.dns_zone)
+  api_fqdn          = format("%s-private.%s.", var.cluster_name, var.dns_zone)
+  etcd_fqdn         = [for i, d in var.controllers_private_ipv4 : format("%s-etcd%d.%s.", var.cluster_name, i, var.dns_zone)]
+
+  dns_entries = concat(
+    [
+      # apiserver public
+      {
+        name    = local.api_external_fqdn,
+        type    = "A",
+        ttl     = 300,
+        records = var.controllers_public_ipv4
+      },
+      # apiserver private
+      {
+        name    = local.api_fqdn,
+        type    = "A",
+        ttl     = 300,
+        records = var.controllers_private_ipv4
+      },
+    ],
+    # etcd
+    [
+      for index, i in var.controllers_private_ipv4 :
+      {
+        name    = local.etcd_fqdn[index],
+        type    = "A",
+        ttl     = 300,
+        records = [i],
+      }
+    ],
+  )
+}
+
+output "entries" {
+  value = local.dns_entries
+}

assets/lokomotive-kubernetes/dns/manual/require.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12.0"
+}

assets/lokomotive-kubernetes/dns/manual/variables.tf

@@ -0,0 +1 @@
+../shared-variables.tf

assets/lokomotive-kubernetes/dns/route53/main.tf

@@ -1,27 +1,36 @@
-variable "entries" {
-  type = list(
-    object({
-      name    = string
-      type    = string
-      ttl     = number
-      records = list(string)
-    })
-  )
+provider "aws" {
+  # The Route 53 service doesn't need a specific region to operate, however
+  # the AWS Terraform provider needs it and the documentation suggests to use
+  # "us-east-1": https://docs.aws.amazon.com/general/latest/gr/r53.html.
+  region = "us-east-1"
 }
 
-variable "aws_zone_id" {
-  type        = string
-  description = "AWS Route53 DNS Zone ID (e.g. Z3PAABBCFAKEC0)"
+data "aws_route53_zone" "selected" {
+  name = "${var.dns_zone}."
 }
 
-resource "aws_route53_record" "dns-records" {
-  count = length(var.entries)
+resource "aws_route53_record" "apiserver_public" {
+  zone_id = data.aws_route53_zone.selected.zone_id
+  name    = format("%s.%s.", var.cluster_name, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  records = var.controllers_public_ipv4
+}
+
+resource "aws_route53_record" "apiserver_private" {
+  zone_id = data.aws_route53_zone.selected.zone_id
+  name    = format("%s-private.%s.", var.cluster_name, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  records = var.controllers_private_ipv4
+}
 
-  # Route53 DNS Zone where record should be created
-  zone_id = var.aws_zone_id
+resource "aws_route53_record" "etcd" {
+  count = length(var.controllers_private_ipv4)
 
-  name    = var.entries[count.index].name
-  type    = var.entries[count.index].type
-  ttl     = var.entries[count.index].ttl
-  records = var.entries[count.index].records
+  zone_id = data.aws_route53_zone.selected.zone_id
+  name    = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)
+  type    = "A"
+  ttl     = 300
+  records = [var.controllers_private_ipv4[count.index]]
 }

assets/lokomotive-kubernetes/dns/route53/require.tf

@@ -1,5 +1,3 @@
-# Terraform version and plugin versions
-
 terraform {
   required_version = ">= 0.12.0"
 

assets/lokomotive-kubernetes/dns/route53/variables.tf

@@ -0,0 +1 @@
+../shared-variables.tf

assets/lokomotive-kubernetes/dns/shared-variables.tf

@@ -0,0 +1,22 @@
+# This file contains variables which are shared among all modules in this directory. Its purpose is
+# to reduce duplication and assist in enforcing a common "interface" for all the modules.
+
+variable "cluster_name" {
+  type        = string
+  description = "Unique cluster name (prepended to dns_zone)"
+}
+
+variable "controllers_public_ipv4" {
+  type        = list(string)
+  description = "Public IPv4 addresses of all the controllers in the cluster"
+}
+
+variable "controllers_private_ipv4" {
+  type        = list(string)
+  description = "Private IPv4 addresses of all the controllers in the cluster"
+}
+
+variable "dns_zone" {
+  type        = string
+  description = "Zone name under which records should be created (e.g. example.com)"
+}

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/bootkube.tf

@@ -3,12 +3,12 @@ module "bootkube" {
   cluster_name = var.cluster_name
 
   # Cannot use cyclic dependencies on controllers or their DNS records
-  api_servers          = [local.api_fqdn]
-  api_servers_external = [local.api_external_fqdn]
-  etcd_servers         = local.etcd_fqdn
+  api_servers          = [format("%s-private.%s", var.cluster_name, var.dns_zone)]
+  api_servers_external = [format("%s.%s", var.cluster_name, var.dns_zone)]
+  etcd_servers         = [for i, d in packet_device.controllers : format("%s-etcd%d.%s", var.cluster_name, i, var.dns_zone)]
   asset_dir            = var.asset_dir
   network_mtu          = var.network_mtu
-  etcd_endpoints       = local.etcd_endpoints
+  etcd_endpoints       = packet_device.controllers.*.access_private_ipv4
 
   # Select private Packet NIC by using the can-reach Calico autodetection option with the first
   # host in our private CIDR.

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -42,14 +42,13 @@ systemd:
         [Unit]
         Description=Wait for DNS entries
         Wants=systemd-resolved.service
-        Before=kubelet.service
+        Before=kubelet.service etcd-member.service bootkube.service
         [Service]
         Type=oneshot
         RemainAfterExit=true
-        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done; /opt/wait-for-dns ${dns_zone} ${cluster_name}-private 3600'
         [Install]
-        RequiredBy=kubelet.service
-        RequiredBy=etcd-member.service
+        RequiredBy=kubelet.service etcd-member.service bootkube.service
     - name: create-etcd-config.service
       # This service will extract value of private interface from the env var file `/run/metadata/flatcar`.
       # And then assign it to the variables that which will be stored in file `/etc/kubernetes/etcd.config`,
@@ -323,6 +322,64 @@ storage:
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
           EOF
+    - path: /opt/wait-for-dns
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash
+          # TODO: Workaround for https://github.com/flatcar-linux/Flatcar/issues/123.
+          function dig {
+              docker run -i --rm quay.io/kinvolk/alpine-dig:3.9.6 dig "$@" 2>/dev/null
+          }
+          if [[ $# -ne 3 ]]; then
+              echo "Usage: $0 <zone> <record> <max_attempts>"
+              exit 1
+          fi
+          zone=$1
+          record=$2
+          max_attempts=$3
+          echo "Figuring out the nameservers for $zone"
+          nameservers=""
+          counter=0
+          while [[ $counter -lt $max_attempts ]]; do
+              out=$(dig +short +timeout=2 "$zone" ns)
+              ret=$?
+              if [[ $ret -eq 0 && "$out" != "" ]]; then
+                  nameservers=$out
+                  break
+              fi
+              echo "Failed with exit code $ret: $out"
+              sleep 1
+              counter=$((counter+1))
+          done
+          if [[ "$nameservers" == "" ]]; then
+              echo "Could not resolve nameservers for $zone"
+              exit 1
+          fi
+          for ns in $nameservers; do
+              echo "Polling $ns for $record.$zone..."
+              counter=0
+              ok=false
+              while [[ $counter -lt $max_attempts ]]; do
+                  out=$(dig +short +timeout=2 @"$ns" "$record"."$zone" a)
+                  ret=$?
+                  if [[ $ret -eq 0 && "$out" != "" ]]; then
+                      echo "Looks good!"
+                      ok=true
+                      break
+                  fi
+                  echo "Not available yet"
+                  sleep 1
+                  counter=$((counter+1))
+              done
+              if ! $ok; then
+                  echo "$record.$zone didn't become available within the allowed time"
+                  exit 1
+              fi
+          done
+          echo "$record.$zone is available on all nameservers"
+          exit 0
 passwd:
   users:
   - name: core

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/controllers.tf

@@ -1,45 +1,3 @@
-locals {
-  api_external_fqdn = format("%s.%s.", var.cluster_name, var.dns_zone)
-  api_fqdn          = format("%s-private.%s.", var.cluster_name, var.dns_zone)
-  etcd_fqdn         = [for index, device in packet_device.controllers : format("%s-etcd%d.%s.", var.cluster_name, index, var.dns_zone)]
-  etcd_endpoints    = packet_device.controllers.*.access_private_ipv4
-
-  dns_entries = concat(
-    [
-      # apiserver public
-      {
-        name    = local.api_external_fqdn,
-        type    = "A",
-        ttl     = 300,
-        records = packet_device.controllers.*.access_public_ipv4,
-      },
-      # apiserver private
-      {
-        name    = local.api_fqdn,
-        type    = "A",
-        ttl     = 300,
-        records = packet_device.controllers.*.access_private_ipv4,
-      },
-    ],
-    # etcd
-    [
-      for index, device in packet_device.controllers :
-      {
-        name    = local.etcd_fqdn[index],
-        type    = "A",
-        ttl     = 300,
-        records = [device.access_private_ipv4],
-      }
-    ],
-  )
-}
-
-resource null_resource "dns_entries" {
-  triggers = {
-    value = jsonencode(local.dns_entries)
-  }
-}
-
 resource "packet_device" "controllers" {
   count            = var.controller_count
   hostname         = "${var.cluster_name}-controller-${count.index}"
@@ -95,6 +53,8 @@ data "ct_config" "controller-ignitions" {
     k8s_dns_service_ip    = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix = var.cluster_domain_suffix
     controller_count      = var.controller_count
+    dns_zone              = var.dns_zone
+    cluster_name          = var.cluster_name
 
     # we need to prepend a prefix 'docker://' for arm64, because arm64 images
     # on quay prevent us from downloading ACI correctly.

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/outputs.tf

@@ -6,10 +6,6 @@ output "kubeconfig" {
   value = module.bootkube.kubeconfig-kubelet
 }
 
-output "dns_entries" {
-  value = local.dns_entries
-}
-
 # values.yaml content for all deployed charts.
 output "pod-checkpointer_values" {
   value = module.bootkube.pod-checkpointer_values
@@ -36,3 +32,11 @@ output "calico_values" {
 output "device_ids" {
   value = packet_device.controllers.*.id
 }
+
+output "controllers_public_ipv4" {
+  value = packet_device.controllers.*.access_public_ipv4
+}
+
+output "controllers_private_ipv4" {
+  value = packet_device.controllers.*.access_private_ipv4
+}