fix(kuma-cp) enable secrets support for Gateway resources (#2953)

Add the secrets generator to the Gateway proxy template. This fixes
mTLS between the gageway proxy and upstream services in the mesh.

Signed-off-by: James Peach <james.peach@konghq.com>
(cherry picked from commit 68d5186)

# Conflicts:
#	test/framework/interface.go
#	test/framework/kumactl.go
#	test/framework/universal_cluster.go

.circleci/config.yml

@@ -956,7 +956,7 @@ jobs:
     - attach_workspace:
         at: build
     - setup_remote_docker:
-        version: 19.03.12
+        version: 20.10.7
     - run:
         name: Build Docker images
         command: |
@@ -986,7 +986,7 @@ jobs:
     - attach_workspace:
         at: build
     - setup_remote_docker:
-        version: 19.03.12
+        version: 20.10.7
     - run:
         name: Build kumactl's Docker image
         command: |

mk/e2e.new.mk

@@ -138,8 +138,5 @@ test/e2e/debug-universal: build/kumactl images/test
 
 .PHONY: test/e2e
 test/e2e: build/kumactl images test/e2e/k8s/start
-	$(MAKE) test/e2e/test || \
-	(ret=$$?; \
-	$(MAKE) test/e2e/k8s/stop && \
-	exit $$ret)
+	$(MAKE) test/e2e/test || (ret=$$?; $(MAKE) test/e2e/k8s/stop && exit $$ret)
 	$(MAKE) test/e2e/k8s/stop

pkg/plugins/runtime/gateway/plugin.go

@@ -45,6 +45,7 @@ func NewProxyProfile(manager manager.ReadOnlyResourceManager) generator.Resource
 	return generator.CompositeResourceGenerator{
 		generator.AdminProxyGenerator{},
 		generator.PrometheusEndpointGenerator{},
+		generator.SecretsProxyGenerator{},
 		generator.TracingProxyGenerator{},
 		generator.TransparentProxyGenerator{},
 		generator.DNSGenerator{},

test/dockerfiles/Dockerfile.universal

@@ -1,7 +1,7 @@
 # using Envoy's base to copy the Envoy binary
 FROM envoyproxy/envoy:v1.18.4 as envoy
 
-FROM ubuntu:20.04
+FROM ubuntu:21.04
 
 RUN mkdir /kuma
 RUN echo "# use this file to override default configuration of \`kuma-cp\`" > /kuma/kuma-cp.conf \

test/e2e/gateway/gateway_universal.go

@@ -12,6 +12,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	config_core "github.com/kumahq/kuma/pkg/config/core"
 	"github.com/kumahq/kuma/test/e2e/trafficroute/testutil"
 	. "github.com/kumahq/kuma/test/framework"
@@ -74,26 +75,29 @@ networking:
 		}
 	}
 
-	BeforeEach(func() {
+	// DeployCluster creates a universal Kuma cluster using the
+	// provided options, installing an echo service as well as a
+	// gateway and a client container to send HTTP requests.
+	DeployCluster := func(opt ...KumaDeploymentOption) {
 		cluster = NewUniversalCluster(NewTestingT(), Kuma1, Silent)
 		Expect(cluster).ToNot(BeNil())
 
-		deployOpts := append(KumaUniversalDeployOpts, WithVerbose())
+		opt = append(opt, WithVerbose())
 
 		err := NewClusterSetup().
-			Install(Kuma(config_core.Standalone, deployOpts...)).
+			Install(Kuma(config_core.Standalone, opt...)).
 			Install(GatewayClientUniversal("gateway-client")).
 			Install(EchoServerUniversal("echo-server")).
 			Install(GatewayProxyUniversal("gateway-proxy")).
 			Setup(cluster)
 
-		// OK, this is messed up. The makefile rule that builds the
-		// kuma-universal:latest image that is used for e2e tests
-		// always rebuilds Kuma with Gateway disabled. This means
-		// that by default, no Gateway tests will work, so we use the
-		// `WithKumactlFlow` option to detect the unsupported gateway
-		// type early (when kumactl creates the dataplane resource),
-		// and skip the remaining tests.
+		// The makefile rule that builds the kuma-universal:latest image
+		// that is used for e2e tests by default rebuilds Kuma with Gateway
+		// disabled. This means that unless BUILD_WITH_EXPERIMENTAL_GATEWAY=Y is
+		// set persistently in the environment, Gateway will not be supported.
+		// We use the `WithKumactlFlow` option to detect the unsupported gateway
+		// type early (when kumactl creates the dataplane resource), and skip
+		// the remaining tests.
 		var shellErr *shell.ErrWithCmdOutput
 		if errors.As(err, &shellErr) {
 			if strings.Contains(shellErr.Output.Combined(), "unsupported gateway type") {
@@ -103,16 +107,12 @@ networking:
 
 		// Otherwise, we expect the cluster build to succeed.
 		Expect(err).To(Succeed())
+	}
 
+	// Before each test, verify the cluster is up and stable.
+	JustBeforeEach(func() {
 		Expect(cluster.VerifyKuma()).To(Succeed())
 
-		// TODO(jpeach) For how the default traffic route make
-		// the gateway generate invalid clusters. Remove this when
-		// we yank TrafficRoute support.
-		Expect(
-			cluster.GetKumactlOptions().KumactlDelete("traffic-route", "route-all-default", "default"),
-		).To(Succeed())
-
 		// Synchronize on the dataplanes coming up.
 		Eventually(func(g Gomega) {
 			dataplanes, err := cluster.GetKumactlOptions().KumactlList("dataplanes", "default")
@@ -121,11 +121,8 @@ networking:
 		}, "60s", "1s").Should(Succeed())
 	})
 
-	E2EAfterEach(func() {
-		Expect(cluster.DismissCluster()).ToNot(HaveOccurred())
-	})
-
-	It("should proxy simple HTTP requests", func() {
+	// Before each test, install the gateway and routes.
+	JustBeforeEach(func() {
 		Expect(
 			cluster.GetKumactlOptions().KumactlApplyFromString(`
 type: Gateway
@@ -172,11 +169,19 @@ conf:
 		gateways, err := cluster.GetKumactlOptions().KumactlList("gateways", "default")
 		Expect(err).To(Succeed())
 		Expect(gateways).To(ContainElement("edge-gateway"))
+	})
 
+	E2EAfterEach(func() {
+		Expect(cluster.DismissCluster()).ToNot(HaveOccurred())
+	})
+
+	// ProxySimpleRequests tests that basic HTTP requests are proxied to a service.
+	ProxySimpleRequests := func() {
 		Eventually(func(g Gomega) {
-			host := net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080")
-			p := path.Join("test", url.PathEscape(GinkgoT().Name()))
-			target := fmt.Sprintf("http://%s/%s", host, p)
+			target := fmt.Sprintf("http://%s/%s",
+				net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080"),
+				path.Join("test", url.PathEscape(GinkgoT().Name())),
+			)
 
 			response, err := testutil.CollectResponse(
 				cluster, "gateway-client", target,
@@ -187,5 +192,56 @@ conf:
 			g.Expect(response.Instance).To(Equal("universal"))
 			g.Expect(response.Received.Headers["Host"]).To(ContainElement("example.kuma.io"))
 		}, "30s", "1s").Should(Succeed())
+	}
+
+	Context("when mTLS is disabled", func() {
+		BeforeEach(func() {
+			DeployCluster(KumaUniversalDeployOpts...)
+		})
+
+		It("should proxy simple HTTP requests", ProxySimpleRequests)
+	})
+
+	Context("when mTLS is enabled", func() {
+		BeforeEach(func() {
+			mtls := WithMeshUpdate("default", func(mesh *mesh_proto.Mesh) *mesh_proto.Mesh {
+				mesh.Mtls = &mesh_proto.Mesh_Mtls{
+					EnabledBackend: "builtin",
+					Backends: []*mesh_proto.CertificateAuthorityBackend{
+						{Name: "builtin", Type: "builtin"},
+					},
+				}
+				return mesh
+			})
+
+			DeployCluster(append(KumaUniversalDeployOpts, mtls)...)
+		})
+
+		It("should proxy simple HTTP requests", ProxySimpleRequests)
+
+		// In mTLS mode, only the presence of TrafficPermission rules allow services to receive
+		// traffic, so removing the permission should cause requests to fail. We use this to
+		// prove that mTLS is enabled
+		It("should fail without TrafficPermission", func() {
+			Expect(
+				cluster.GetKumactlOptions().KumactlDelete(
+					"traffic-permission", "allow-all-default", "default"),
+			).ToNot(HaveOccurred())
+
+			Eventually(func(g Gomega) {
+				target := fmt.Sprintf("http://%s/%s",
+					net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080"),
+					path.Join("test", url.PathEscape(GinkgoT().Name())),
+				)
+
+				status, err := testutil.CollectFailure(
+					cluster, "gateway-client", target,
+					testutil.WithHeader("Host", "example.kuma.io"),
+				)
+
+				g.Expect(err).To(Succeed())
+				g.Expect(status.ResponseCode).To(Equal(503))
+			}, "30s", "1s").Should(Succeed())
+		})
 	})
 }

test/e2e/trafficroute/testutil/collect.go

@@ -3,9 +3,11 @@ package testutil
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"sync"
 
 	"github.com/kballard/go-shellquote"
+	"github.com/pkg/errors"
 
 	"github.com/kumahq/kuma/test/framework"
 	"github.com/kumahq/kuma/test/server/types"
@@ -71,6 +73,75 @@ func CollectResponse(cluster framework.Cluster, source, destination string, fn .
 	return *response, nil
 }
 
+// FailureResponse is the JSON output for a Curl command. Note that the available
+// fields depend on the Curl version, which must be at least 7.70.0 for this feature.
+//
+// See https://curl.se/docs/manpage.html#-w.
+type FailureResponse struct {
+	Errormsg string `json:"errormsg"`
+	Exitcode int    `json:"exitcode"`
+
+	ResponseCode int    `json:"response_code"`
+	Method       string `json:"method"`
+	Scheme       string `json:"scheme"`
+	ContentType  string `json:"content_type"`
+	URL          string `json:"url"`
+	EffectiveURL string `json:"url_effective"`
+}
+
+// CollectFailure runs Curl to fetch a URL that is expected to fail. The
+// Curl JSON output is returned so the caller can inspect the failure to
+// see whether it was what was expected.
+func CollectFailure(cluster framework.Cluster, source, destination string, fn ...CollectResponsesOptsFn) (FailureResponse, error) {
+	opts := DefaultCollectResponsesOpts()
+	for _, f := range fn {
+		f(&opts)
+	}
+
+	cmd := []string{
+		"curl",
+		"--request", opts.Method,
+		"--max-time", "3",
+		"--silent",               // Suppress human-readable errors.
+		"--write-out", "%{json}", // Write JSON result. Requires curl 7.70.0, April 2020.
+		// Silence output so that we don't try to parse it. A future refactor could try to address this
+		// by using "%{stderr}%{json}", but that needs a bit more investigation.
+		"--output", os.DevNull,
+	}
+
+	for key, value := range opts.Headers {
+		cmd = append(cmd, "--header", shellquote.Join(fmt.Sprintf("%s: %s", key, value)))
+	}
+
+	cmd = append(cmd, shellquote.Join(destination))
+	stdout, _, err := cluster.Exec("", "", source, cmd...)
+
+	// 1. If we fail to decode the JSON status, return the JSON error,
+	// but prefer the original error if we have it.
+	empty := FailureResponse{}
+	response := FailureResponse{}
+	if jsonErr := json.Unmarshal([]byte(stdout), &response); jsonErr != nil {
+		// Prefer the original error to a JSON decoding error.
+		if err == nil {
+			return response, jsonErr
+		}
+	}
+
+	// 2. If there was no error response, we still prefer the original
+	// error, but fall back to reporting that the JSON  is missing.
+	if response == empty {
+		if err != nil {
+			return response, err
+		}
+
+		return response, errors.Errorf("empty JSON response from curl: %q", stdout)
+	}
+
+	// 3. Finally, report the JSON status and no execution error
+	// since the JSON contains all the Curl error information.
+	return response, nil
+}
+
 func CollectResponses(cluster framework.Cluster, source, destination string, fn ...CollectResponsesOptsFn) ([]types.EchoResponse, error) {
 	opts := DefaultCollectResponsesOpts()
 	for _, f := range fn {

test/framework/interface.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/testing"
 
+	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	"github.com/kumahq/kuma/pkg/config/core"
 )
 
@@ -43,13 +44,18 @@ type kumaDeploymentOptions struct {
 	cpReplicas           int
 	hdsDisabled          bool
 	runPostgresMigration bool
+
+	// Functions to apply to each mesh after the control plane
+	// is provisioned.
+	meshUpdateFuncs map[string][]func(*mesh_proto.Mesh) *mesh_proto.Mesh
 }
 
 func (k *kumaDeploymentOptions) apply(opts ...KumaDeploymentOption) {
 	// Set defaults.
 	k.isipv6 = IsIPv6()
 	k.installationMode = KumactlInstallationMode
 	k.env = map[string]string{}
+	k.meshUpdateFuncs = map[string][]func(*mesh_proto.Mesh) *mesh_proto.Mesh{}
 
 	// Apply options.
 	for _, o := range opts {
@@ -256,6 +262,33 @@ func WithGlobalAddress(address string) KumaDeploymentOption {
 	})
 }
 
+<<<<<<< HEAD
+=======
+// WithCtlOpt allows arbitrary options to be passed to kuma, which is important
+// for using test/framework in other libraries where additional options may have
+// been added.
+func WithCtlOpt(name, value string) KumaDeploymentOption {
+	return KumaOptionFunc(func(o *kumaDeploymentOptions) {
+		if o.ctlOpts == nil {
+			o.ctlOpts = map[string]string{}
+		}
+		o.ctlOpts[name] = value
+	})
+}
+
+type MeshUpdateFunc func(mesh *mesh_proto.Mesh) *mesh_proto.Mesh
+
+// WithMeshUpdate registers a function to update the specification
+// for the named mesh. When the control plane implementation creates the
+// mesh, it invokes the function and applies configuration changes to the
+// mesh object.
+func WithMeshUpdate(mesh string, u MeshUpdateFunc) KumaDeploymentOption {
+	return KumaOptionFunc(func(o *kumaDeploymentOptions) {
+		o.meshUpdateFuncs[mesh] = append(o.meshUpdateFuncs[mesh], u)
+	})
+}
+
+>>>>>>> 68d51861 (fix(kuma-cp) enable secrets support for Gateway resources (#2953))
 // WithoutDataplane suppresses the automatic configuration of kuma-dp
 // in the application container. This is useful when the test requires a
 // container that is not bound to the mesh.