Support Transparent Proxy for peering.

hashicorp · Jul 26, 2022 · 64a4b04 · 64a4b04
1 parent 02f8d68
commit 64a4b04
Show file tree

Hide file tree

Showing 7 changed files with 192 additions and 15 deletions.
diff --git a/acceptance/tests/peering/peering_connect_namespaces_test.go b/acceptance/tests/peering/peering_connect_namespaces_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/go-version"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -78,7 +79,7 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 
 				"global.image": "thisisnotashwin/consul@sha256:446aad6e02f66e3027756dfc0d34e8e6e2b11ac6ec5637b134b34644ca7cda64",
 
-				"global.tls.enabled":           "false",
+				"global.tls.enabled":           "true",
 				"global.tls.httpsOnly":         strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
 				"global.tls.enableAutoEncrypt": strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
 
@@ -142,6 +143,13 @@ func TestPeering_ConnectNamespaces(t *testing.T) {
 				k8s.KubectlDelete(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml")
 			})
 
+			// Ensure the secret is created.
+			retry.Run(t, func(r *retry.R) {
+				acceptorSecretResourceVersion, err := k8s.RunKubectlAndGetOutputE(t, staticClientPeerClusterContext.KubectlOptions(t), "get", "peeringacceptor", "server", "-o", "jsonpath={.status.secret.resourceVersion}")
+				require.NoError(r, err)
+				require.NotEmpty(r, acceptorSecretResourceVersion)
+			})
+
 			// Copy secret from client peer to server peer.
 			k8s.CopySecret(t, staticClientPeerClusterContext, staticServerPeerClusterContext, "api-token")
 

diff --git a/acceptance/tests/peering/peering_connect_test.go b/acceptance/tests/peering/peering_connect_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/go-version"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -51,7 +52,7 @@ func TestPeering_Connect(t *testing.T) {
 
 				"global.image": "thisisnotashwin/consul@sha256:446aad6e02f66e3027756dfc0d34e8e6e2b11ac6ec5637b134b34644ca7cda64",
 
-				"global.tls.enabled":           "false",
+				"global.tls.enabled":           "true",
 				"global.tls.httpsOnly":         strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
 				"global.tls.enableAutoEncrypt": strconv.FormatBool(c.ACLsAndAutoEncryptEnabled),
 
@@ -110,9 +111,13 @@ func TestPeering_Connect(t *testing.T) {
 			helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() {
 				k8s.KubectlDelete(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml")
 			})
-			acceptorSecretResourceVersion, err := k8s.RunKubectlAndGetOutputE(t, staticClientPeerClusterContext.KubectlOptions(t), "get", "peeringacceptor", "server", "-o", "jsonpath={.status.secret.resourceVersion}")
-			require.NoError(t, err)
-			require.NotEmpty(t, acceptorSecretResourceVersion)
+
+			// Ensure the secret is created.
+			retry.Run(t, func(r *retry.R) {
+				acceptorSecretResourceVersion, err := k8s.RunKubectlAndGetOutputE(t, staticClientPeerClusterContext.KubectlOptions(t), "get", "peeringacceptor", "server", "-o", "jsonpath={.status.secret.resourceVersion}")
+				require.NoError(r, err)
+				require.NotEmpty(r, acceptorSecretResourceVersion)
+			})
 
 			// Copy secret from client peer to server peer.
 			k8s.CopySecret(t, staticClientPeerClusterContext, staticServerPeerClusterContext, "api-token")

diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml
@@ -283,6 +283,31 @@ spec:
                 {{- end }}
                 -hcl='leave_on_terminate = true' \
                 {{- if .Values.global.tls.enabled }}
+                {{- if .Values.global.peering.enabled }}
+                {{- if .Values.global.secretsBackend.vault.enabled }}
+                -hcl='tls { defaults { ca_file = "/vault/secrets/serverca.crt" }}' \
+                {{- else }}
+                -hcl='tls { defaults { ca_file = "/consul/tls/ca/tls.crt" }}' \
+                {{- end }}
+                {{- if .Values.global.tls.enableAutoEncrypt }}
+                -hcl='auto_encrypt = {tls = true}' \
+                -hcl="auto_encrypt = {ip_san = [\"$HOST_IP\",\"$POD_IP\"]}" \
+                {{- else }}
+                -hcl='tls { defaults { cert_file = "/consul/tls/client/tls.crt" }}' \
+                -hcl='tls { defaults { key_file = "/consul/tls/client/tls.key" }}' \
+                {{- end }}
+                {{- if .Values.global.tls.verify }}
+                -hcl='tls { defaults { verify_outgoing = true }}' \
+                {{- if not .Values.global.tls.enableAutoEncrypt }}
+                -hcl='tls { internal_rpc { verify_incoming = true }}' \
+                -hcl='tls { internal_rpc { verify_server_hostname = true }}' \
+                {{- end }}
+                {{- end }}
+                -hcl='ports { https = 8501 }' \
+                {{- if .Values.global.tls.httpsOnly }}
+                -hcl='ports { http = -1 }' \
+                {{- end }}
+                {{- else}}
                 {{- if .Values.global.secretsBackend.vault.enabled }}
                 -hcl='ca_file = "/vault/secrets/serverca.crt"' \
                 {{- else }}
@@ -307,6 +332,7 @@ spec:
                 -hcl='ports { http = -1 }' \
                 {{- end }}
                 {{- end }}
+                {{- end }}
                 {{- if .Values.client.grpc }}
                 -hcl='ports { grpc = 8502 }' \
                 {{- end }}

diff --git a/charts/consul/templates/server-config-configmap.yaml b/charts/consul/templates/server-config-configmap.yaml
@@ -87,14 +87,52 @@ data:
   {{- if .Values.global.tls.enabled }}
   tls-config.json: |-
     {
-       {{- if .Values.global.secretsBackend.vault.enabled }}
-       "ca_file": "/vault/secrets/serverca.crt",
-       "cert_file": "/vault/secrets/servercert.crt",
-       "key_file": "/vault/secrets/servercert.key",
+      {{- if .Values.global.peering.enabled }}
+      "tls": {
+        {{- if .Values.global.tls.verify }}
+        "internal_rpc": {
+          "verify_incoming": true,
+          "verify_server_hostname": true
+        },
+        "grpc": {
+          "verify_incoming": false
+        },
+        {{- end }}
+        "defaults": {
+          {{- if .Values.global.tls.verify }}
+          "verify_outgoing": true,
+          {{- end }}
+          {{- if .Values.global.secretsBackend.vault.enabled }}
+          "ca_file": "/vault/secrets/serverca.crt",
+          "cert_file": "/vault/secrets/servercert.crt",
+          "key_file": "/vault/secrets/servercert.key"
+          {{- else }}
+          "ca_file": "/consul/tls/ca/tls.crt",
+          "cert_file": "/consul/tls/server/tls.crt",
+          "key_file": "/consul/tls/server/tls.key"
+          {{- end }}
+        }
+      },
+      {{- if .Values.global.tls.enableAutoEncrypt }}
+      "auto_encrypt": {
+        "allow_tls": true
+      },
+      {{- end }}
+      "ports": {
+        {{- if .Values.global.tls.httpsOnly }}
+        "http": -1,
+        {{- end }}
+        "https": 8501
+      }
+      {{- else }}
+      {{- if .Values.global.secretsBackend.vault.enabled }}
+      "ca_file": "/vault/secrets/serverca.crt",
+      "cert_file": "/vault/secrets/servercert.crt",
+      "key_file": "/vault/secrets/servercert.key",
        {{- else }}
-       "ca_file": "/consul/tls/ca/tls.crt",
-       "cert_file": "/consul/tls/server/tls.crt",
-       "key_file": "/consul/tls/server/tls.key",
+      "ca_file": "/consul/tls/ca/tls.crt",
+      "cert_file": "/consul/tls/server/tls.crt",
+      "key_file": "/consul/tls/server/tls.key",
        {{- end }}
       {{- if .Values.global.tls.enableAutoEncrypt }}
       "auto_encrypt": {
@@ -110,9 +148,9 @@ data:
         {{- if .Values.global.tls.httpsOnly }}
         "http": -1,
         {{- end }}
-        "https": 8501,
-        "grpc":8503
+        "https": 8501
       }
+      {{- end }}
     }
   {{- end }}
   {{- if .Values.ui.enabled }}

diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
@@ -296,7 +296,6 @@ spec:
                 -config-dir=/consul/userconfig/{{ .name }} \
                 {{- end }}
                 {{- end }}
-                -hcl='ports { grpc = 8503 }' \
                 -config-file=/consul/extra-config/extra-from-values.json
           volumeMounts:
             - name: data-{{ .Release.Namespace | trunc 58 | trimSuffix "-" }}

diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
@@ -899,6 +899,27 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "client/DaemonSet: sets verify_* flags to true by default when global.tls.enabled and global.peering.enabled" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/client-daemonset.yaml  \
+      --set 'global.tls.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command | join(" ")' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo $command | jq -r '. | contains("tls { internal_rpc { verify_incoming = true }}")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  actual=$(echo $command | jq -r '. | contains("tls { defaults { verify_outgoing = true }}")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  actual=$(echo $command | jq -r '. | contains("tls { internal_rpc { verify_server_hostname = true }}")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 @test "client/DaemonSet: doesn't set the verify_* flags when global.tls.enabled is true and global.tls.verify is false" {
   cd `chart_dir`
   local command=$(helm template \

diff --git a/charts/consul/test/unit/server-config-configmap.bats b/charts/consul/test/unit/server-config-configmap.bats
@@ -700,6 +700,39 @@ load _helpers
   [ "${actual}" = '{"http":-1,"https":8501}' ]
 }
 
+@test "server/ConfigMap: sets correct default configuration when global.tls.enabled and global.peering.enabled" {
+  cd `chart_dir`
+  local config=$(helm template \
+      -s templates/server-config-configmap.yaml \
+      --set 'global.tls.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.data["tls-config.json"]' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo $config | jq -r .tls.defaults.ca_file | tee /dev/stderr)
+  [ "${actual}" = "/consul/tls/ca/tls.crt" ]
+
+  actual=$(echo $config | jq -r .tls.defaults.cert_file | tee /dev/stderr)
+  [ "${actual}" = "/consul/tls/server/tls.crt" ]
+
+  actual=$(echo $config | jq -r .tls.defaults.key_file | tee /dev/stderr)
+  [ "${actual}" = "/consul/tls/server/tls.key" ]
+
+  actual=$(echo $config | jq -r .tls.internal_rpc.verify_incoming | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  actual=$(echo $config | jq -r .tls.defaults.verify_outgoing | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  actual=$(echo $config | jq -r .tls.internal_rpc.verify_server_hostname | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  actual=$(echo $config | jq -c .ports | tee /dev/stderr)
+  [ "${actual}" = '{"http":-1,"https":8501}' ]
+}
+
 @test "server/ConfigMap: doesn't set verify_* configuration to true when global.tls.enabled and global.tls.verify is false" {
   cd `chart_dir`
   local config=$(helm template \
@@ -720,6 +753,25 @@ load _helpers
   [ "${actual}" = "null" ]
 }
 
+@test "server/ConfigMap: doesn't set verify_* configuration to true when global.tls.enabled and global.peering.enabled and global.tls.verify is false" {
+  cd `chart_dir`
+  local config=$(helm template \
+      -s templates/server-config-configmap.yaml \
+      --set 'global.tls.enabled=true' \
+      --set 'global.peering.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.tls.verify=false' \
+      . | tee /dev/stderr |
+      yq -r '.data["tls-config.json"]' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo $config | jq -r .tls.internal_rpc | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  actual=$(echo $config | jq -r .tls.defaults.verify_outgoing | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
 @test "server/ConfigMap: HTTP port is not set in when httpsOnly is false" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -774,6 +826,34 @@ load _helpers
   [ "${actual}" = "/vault/secrets/servercert.key" ]
 }
 
+@test "server/ConfigMap: sets TLS file paths to point to vault secrets when Vault is enabled and global.peering.enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/server-config-configmap.yaml  \
+    --set 'global.tls.enabled=true' \
+    --set 'global.peering.enabled=true' \
+    --set 'connectInject.enabled=true' \
+    --set 'global.tls.enableAutoEncrypt=true' \
+    --set 'global.datacenter=dc2' \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=test' \
+    --set 'global.secretsBackend.vault.consulServerRole=foo' \
+    --set 'global.secretsBackend.vault.consulCARole=test' \
+    --set 'global.tls.caCert.secretName=pki_int/cert/ca' \
+    --set 'server.serverCert.secretName=pki_int/issue/test' \
+    . | tee /dev/stderr |
+    yq -r '.data["tls-config.json"]' | tee /dev/stderr)
+
+  local actual=$(echo $object | jq -r .tls.defaults.ca_file | tee /dev/stderr)
+  [ "${actual}" = "/vault/secrets/serverca.crt" ]
+
+  local actual=$(echo $object | jq -r .tls.defaults.cert_file | tee /dev/stderr)
+  [ "${actual}" = "/vault/secrets/servercert.crt" ]
+
+  local actual=$(echo $object | jq -r .tls.defaults.key_file | tee /dev/stderr)
+  [ "${actual}" = "/vault/secrets/servercert.key" ]
+}
+
 @test "server/ConfigMap: when global.metrics.enableAgentMetrics=true, sets telemetry config" {
   cd `chart_dir`
   local actual=$(helm template \