Use public cloud for digital ocean and secure with TLS

"Private network" on digital ocean is really a shared private network. It means all other hosts in datacenter, even of other users, can access deployed nodes on digitalocean. It means deploying on public ips is not less secure than deploying on private ones. In this setup we additionaly secure etcd with wildcard tls certs. As an additional bonus, we can not deploy servers across datacenters. Also, we generate initial ssh key for digital ocean dynamically, for better security, and export it to "id_rsa" file for easy ssh access.
Capgemini · Feb 28, 2016 · b1e98cf · b1e98cf
1 parent d529c5a
commit b1e98cf
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 36 deletions.
diff --git a/terraform/digitalocean/.gitignore b/terraform/digitalocean/.gitignore
@@ -0,0 +1,3 @@
+id_rsa
+id_rsa.pub
+etcd_discovery_url.txt
diff --git a/terraform/digitalocean/keypair/main.tf b/terraform/digitalocean/keypair/main.tf
diff --git a/terraform/digitalocean/main.tf b/terraform/digitalocean/main.tf
@@ -1,5 +1,4 @@
 variable "do_token" {}
-variable "public_key_file" { default = "~/.ssh/digitalocean.pub" }
 variable "region" { default = "lon1" }
 variable "masters" { default = "3" }
 variable "slaves" { default = "1" }
@@ -13,11 +12,82 @@ provider "digitalocean" {
   token = "${var.do_token}"
 }
 
-# ssh keypair for instances
-module "do-keypair" {
-  source = "./keypair"
+resource "tls_private_key" "ssh" {
+  algorithm = "RSA"
+}
+
+resource "digitalocean_ssh_key" "default" {
+  name       = "skybase"
+  public_key = "${tls_private_key.ssh.public_key_openssh}"
+}
+
+# Export ssh key so we can login with core@instance -i id_rsa
+resource "null_resource" "keys" {
+  depends_on = ["tls_private_key.ssh"]
+
+  provisioner "local-exec" {
+    command = "echo '${tls_private_key.ssh.private_key_pem}' > ${path.module}/id_rsa && chmod 600 ${path.module}/id_rsa"
+  }
+}
+
+resource "tls_private_key" "ca" {
+  algorithm = "RSA"
+}
+
+resource "tls_self_signed_cert" "ca" {
+  key_algorithm = "RSA"
+  private_key_pem = "${tls_private_key.ca.private_key_pem}"
+
+  subject {
+    common_name = "*"
+    organization = "skybase"
+  }
+
+  allowed_uses = [
+    "key_encipherment",
+    "cert_signing",
+    "server_auth",
+    "client_auth"
+  ]
+
+  validity_period_hours = 43800
 
-  public_key_file = "${var.public_key_file}"
+  early_renewal_hours = 720
+
+  is_ca_certificate = true
+}
+
+resource "tls_private_key" "etcd" {
+  algorithm = "RSA"
+}
+
+resource "tls_cert_request" "etcd" {
+  key_algorithm = "RSA"
+
+  private_key_pem = "${tls_private_key.etcd.private_key_pem}"
+
+  subject {
+    common_name = "*"
+    organization = "etcd"
+  }
+}
+
+resource "tls_locally_signed_cert" "etcd" {
+  cert_request_pem = "${tls_cert_request.etcd.cert_request_pem}"
+
+  ca_key_algorithm = "RSA"
+  ca_private_key_pem = "${tls_private_key.ca.private_key_pem}"
+  ca_cert_pem = "${tls_self_signed_cert.ca.cert_pem}"
+
+  validity_period_hours = 43800
+
+  early_renewal_hours = 720
+
+  allowed_uses = [
+    "key_encipherment",
+    "server_auth",
+    "client_auth"
+  ]
 }
 
 # Generate an etcd URL for the cluster
@@ -38,6 +108,9 @@ resource "template_file" "master_cloud_init" {
   vars {
     etcd_discovery_url = "${file(var.etcd_discovery_url_file)}"
     size               = "${var.masters + var.slaves}"
+    etcd_ca            = "${replace(tls_self_signed_cert.ca.cert_pem, \"\n\", \"\\n\")}"
+    etcd_cert          = "${replace(tls_locally_signed_cert.etcd.cert_pem, \"\n\", \"\\n\")}"
+    etcd_key           = "${replace(tls_private_key.etcd.private_key_pem, \"\n\", \"\\n\")}"
   }
 }
 
@@ -47,6 +120,9 @@ resource "template_file" "slave_cloud_init" {
   vars {
     etcd_discovery_url = "${file(var.etcd_discovery_url_file)}"
     size               = "${var.masters + var.slaves}"
+    etcd_ca            = "${replace(tls_self_signed_cert.ca.cert_pem, \"\n\", \"\\n\")}"
+    etcd_cert          = "${replace(tls_locally_signed_cert.etcd.cert_pem, \"\n\", \"\\n\")}"
+    etcd_key           = "${replace(tls_private_key.etcd.private_key_pem, \"\n\", \"\\n\")}"
   }
 }
 
@@ -60,7 +136,7 @@ resource "digitalocean_droplet" "mesos-master" {
   private_networking = true
   user_data          = "${template_file.master_cloud_init.rendered}"
   ssh_keys = [
-    "${module.do-keypair.keypair_id}"
+    "${digitalocean_ssh_key.default.id}"
   ]
 }
 
@@ -74,7 +150,7 @@ resource "digitalocean_droplet" "mesos-slave" {
   private_networking = true
   user_data          = "${template_file.slave_cloud_init.rendered}"
   ssh_keys = [
-    "${module.do-keypair.keypair_id}"
+    "${digitalocean_ssh_key.default.id}"
   ]
 }
 

diff --git a/terraform/digitalocean/master-cloud-config.yml.tpl b/terraform/digitalocean/master-cloud-config.yml.tpl
@@ -2,20 +2,35 @@
 
 coreos:
   etcd2:
-    # $private_ipv4 is populated by the cloud provider
-    # we don't have a $public_ipv4 in the private VPC
-    advertise-client-urls: http://$private_ipv4:2379,http://$private_ipv4:4001
-    initial-advertise-peer-urls: http://$private_ipv4:2380
-    # listen on both the official ports and the legacy ports
-    # legacy ports can be omitted if your application doesn't depend on them
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
-    listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001
-    # Discovery is populated by Terraform
     discovery: ${etcd_discovery_url}
+    advertise-client-urls: http://$public_ipv4:2379
+    initial-advertise-peer-urls: http://$public_ipv4:2380
+    listen-client-urls: http://0.0.0.0:2379
+    listen-peer-urls: http://$public_ipv4:2380
   units:
     - name: etcd2.service
       command: start
   update:
     reboot-strategy: best-effort
+write_files:
+  - path: /run/systemd/system/etcd.service.d/30-certificates.conf
+    permissions: 0644
+    content: |
+      [Service]
+      Environment=ETCD_CA_FILE=/etc/ssl/etcd/certs/ca.pem
+      Environment=ETCD_CERT_FILE=/etc/ssl/etcd/certs/etcd.pem
+      Environment=ETCD_KEY_FILE=/etc/ssl/etcd/private/etcd.pem
+      Environment=ETCD_PEER_CA_FILE=/etc/ssl/etcd/certs/ca.pem
+      Environment=ETCD_PEER_CERT_FILE=/etc/ssl/etcd/certs/etcd.pem
+      Environment=ETCD_PEER_KEY_FILE=/etc/ssl/etcd/private/etcd.pem
+  - path: /etc/ssl/etcd/certs/ca.pem
+    permissions: 0644
+    content: "${etcd_ca}"
+  - path: /etc/ssl/etcd/certs/etcd.pem
+    permissions: 0644
+    content: "${etcd_cert}"
+  - path: /etc/ssl/etcd/private/etcd.pem
+    permissions: 0644
+    content: "${etcd_key}"
 manage_etc_hosts: localhost
 role: mesos_masters
diff --git a/terraform/digitalocean/slave-cloud-config.yml.tpl b/terraform/digitalocean/slave-cloud-config.yml.tpl
@@ -2,20 +2,35 @@
 
 coreos:
   etcd2:
-    # $private_ipv4 is populated by the cloud provider
-    # we don't have a $public_ipv4 in the private VPC
-    advertise-client-urls: http://$private_ipv4:2379,http://$private_ipv4:4001
-    initial-advertise-peer-urls: http://$private_ipv4:2380
-    # listen on both the official ports and the legacy ports
-    # legacy ports can be omitted if your application doesn't depend on them
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
-    listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001
-    # Discovery is populated by Terraform
     discovery: ${etcd_discovery_url}
+    advertise-client-urls: http://$public_ipv4:2379
+    initial-advertise-peer-urls: http://$public_ipv4:2380
+    listen-client-urls: http://0.0.0.0:2379
+    listen-peer-urls: http://$public_ipv4:2380
   units:
     - name: etcd2.service
       command: start
   update:
     reboot-strategy: best-effort
+write_files:
+  - path: /run/systemd/system/etcd.service.d/30-certificates.conf
+    permissions: 0644
+    content: |
+      [Service]
+      Environment=ETCD_CA_FILE=/etc/ssl/etcd/certs/ca.pem
+      Environment=ETCD_CERT_FILE=/etc/ssl/etcd/certs/etcd.pem
+      Environment=ETCD_KEY_FILE=/etc/ssl/etcd/private/etcd.pem
+      Environment=ETCD_PEER_CA_FILE=/etc/ssl/etcd/certs/ca.pem
+      Environment=ETCD_PEER_CERT_FILE=/etc/ssl/etcd/certs/etcd.pem
+      Environment=ETCD_PEER_KEY_FILE=/etc/ssl/etcd/private/etcd.pem
+  - path: /etc/ssl/etcd/certs/ca.pem
+    permissions: 0644
+    content: "${etcd_ca}"
+  - path: /etc/ssl/etcd/certs/etcd.pem
+    permissions: 0644
+    content: "${etcd_cert}"
+  - path: /etc/ssl/etcd/private/etcd.pem
+    permissions: 0644
+    content: "${etcd_key}"
 manage_etc_hosts: localhost
 role: mesos_slaves