From dd812df3af70226f535edb4c6fee38847a5a7c52 Mon Sep 17 00:00:00 2001 From: James Peach Date: Fri, 22 Oct 2021 09:53:04 +1100 Subject: [PATCH] fix(kuma-cp) enable secrets support for Gateway resources (#2953) Add the secrets generator to the Gateway proxy template. This fixes mTLS between the gageway proxy and upstream services in the mesh. Signed-off-by: James Peach (cherry picked from commit 68d51861ebd371c85ff053d6ed3951eba21d8506) --- .circleci/config.yml | 4 +- mk/e2e.new.mk | 5 +- pkg/plugins/runtime/gateway/plugin.go | 1 + test/dockerfiles/Dockerfile.universal | 2 +- test/e2e/gateway/gateway_universal.go | 106 +++++++++++++++++----- test/e2e/trafficroute/testutil/collect.go | 71 +++++++++++++++ test/framework/interface.go | 18 ++++ test/framework/k8s_cluster.go | 38 +++++++- test/framework/k8s_controlplane.go | 67 ++++++++++++++ test/framework/kumactl.go | 89 +++++++++++------- test/framework/universal_app.go | 6 +- test/framework/universal_cluster.go | 19 +++- test/framework/universal_controlplane.go | 9 ++ 13 files changed, 366 insertions(+), 69 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cea8d2097231..26483375be7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -956,7 +956,7 @@ jobs: - attach_workspace: at: build - setup_remote_docker: - version: 19.03.12 + version: 20.10.7 - run: name: Build Docker images command: | @@ -986,7 +986,7 @@ jobs: - attach_workspace: at: build - setup_remote_docker: - version: 19.03.12 + version: 20.10.7 - run: name: Build kumactl's Docker image command: | diff --git a/mk/e2e.new.mk b/mk/e2e.new.mk index 7b63a42bd787..d1a46f1b40eb 100644 --- a/mk/e2e.new.mk +++ b/mk/e2e.new.mk @@ -138,8 +138,5 @@ test/e2e/debug-universal: build/kumactl images/test .PHONY: test/e2e test/e2e: build/kumactl images test/e2e/k8s/start - $(MAKE) test/e2e/test || \ - (ret=$$?; \ - $(MAKE) test/e2e/k8s/stop && \ - exit $$ret) + $(MAKE) test/e2e/test || (ret=$$?; $(MAKE) test/e2e/k8s/stop && exit $$ret) $(MAKE) test/e2e/k8s/stop diff --git a/pkg/plugins/runtime/gateway/plugin.go b/pkg/plugins/runtime/gateway/plugin.go index 131b1dfff20a..9dfb586fb259 100644 --- a/pkg/plugins/runtime/gateway/plugin.go +++ b/pkg/plugins/runtime/gateway/plugin.go @@ -45,6 +45,7 @@ func NewProxyProfile(manager manager.ReadOnlyResourceManager) generator.Resource return generator.CompositeResourceGenerator{ generator.AdminProxyGenerator{}, generator.PrometheusEndpointGenerator{}, + generator.SecretsProxyGenerator{}, generator.TracingProxyGenerator{}, generator.TransparentProxyGenerator{}, generator.DNSGenerator{}, diff --git a/test/dockerfiles/Dockerfile.universal b/test/dockerfiles/Dockerfile.universal index 8d7585f8f981..026219fe53f3 100644 --- a/test/dockerfiles/Dockerfile.universal +++ b/test/dockerfiles/Dockerfile.universal @@ -1,7 +1,7 @@ # using Envoy's base to copy the Envoy binary FROM envoyproxy/envoy:v1.18.4 as envoy -FROM ubuntu:20.04 +FROM ubuntu:21.04 RUN mkdir /kuma RUN echo "# use this file to override default configuration of \`kuma-cp\`" > /kuma/kuma-cp.conf \ diff --git a/test/e2e/gateway/gateway_universal.go b/test/e2e/gateway/gateway_universal.go index 4c4e99e71412..e14969d1aa07 100644 --- a/test/e2e/gateway/gateway_universal.go +++ b/test/e2e/gateway/gateway_universal.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" config_core "github.com/kumahq/kuma/pkg/config/core" "github.com/kumahq/kuma/test/e2e/trafficroute/testutil" . "github.com/kumahq/kuma/test/framework" @@ -74,26 +75,29 @@ networking: } } - BeforeEach(func() { + // DeployCluster creates a universal Kuma cluster using the + // provided options, installing an echo service as well as a + // gateway and a client container to send HTTP requests. + DeployCluster := func(opt ...KumaDeploymentOption) { cluster = NewUniversalCluster(NewTestingT(), Kuma1, Silent) Expect(cluster).ToNot(BeNil()) - deployOpts := append(KumaUniversalDeployOpts, WithVerbose()) + opt = append(opt, WithVerbose()) err := NewClusterSetup(). - Install(Kuma(config_core.Standalone, deployOpts...)). + Install(Kuma(config_core.Standalone, opt...)). Install(GatewayClientUniversal("gateway-client")). Install(EchoServerUniversal("echo-server")). Install(GatewayProxyUniversal("gateway-proxy")). Setup(cluster) - // OK, this is messed up. The makefile rule that builds the - // kuma-universal:latest image that is used for e2e tests - // always rebuilds Kuma with Gateway disabled. This means - // that by default, no Gateway tests will work, so we use the - // `WithKumactlFlow` option to detect the unsupported gateway - // type early (when kumactl creates the dataplane resource), - // and skip the remaining tests. + // The makefile rule that builds the kuma-universal:latest image + // that is used for e2e tests by default rebuilds Kuma with Gateway + // disabled. This means that unless BUILD_WITH_EXPERIMENTAL_GATEWAY=Y is + // set persistently in the environment, Gateway will not be supported. + // We use the `WithKumactlFlow` option to detect the unsupported gateway + // type early (when kumactl creates the dataplane resource), and skip + // the remaining tests. var shellErr *shell.ErrWithCmdOutput if errors.As(err, &shellErr) { if strings.Contains(shellErr.Output.Combined(), "unsupported gateway type") { @@ -103,16 +107,12 @@ networking: // Otherwise, we expect the cluster build to succeed. Expect(err).To(Succeed()) + } + // Before each test, verify the cluster is up and stable. + JustBeforeEach(func() { Expect(cluster.VerifyKuma()).To(Succeed()) - // TODO(jpeach) For how the default traffic route make - // the gateway generate invalid clusters. Remove this when - // we yank TrafficRoute support. - Expect( - cluster.GetKumactlOptions().KumactlDelete("traffic-route", "route-all-default", "default"), - ).To(Succeed()) - // Synchronize on the dataplanes coming up. Eventually(func(g Gomega) { dataplanes, err := cluster.GetKumactlOptions().KumactlList("dataplanes", "default") @@ -121,11 +121,8 @@ networking: }, "60s", "1s").Should(Succeed()) }) - E2EAfterEach(func() { - Expect(cluster.DismissCluster()).ToNot(HaveOccurred()) - }) - - It("should proxy simple HTTP requests", func() { + // Before each test, install the gateway and routes. + JustBeforeEach(func() { Expect( cluster.GetKumactlOptions().KumactlApplyFromString(` type: Gateway @@ -172,11 +169,19 @@ conf: gateways, err := cluster.GetKumactlOptions().KumactlList("gateways", "default") Expect(err).To(Succeed()) Expect(gateways).To(ContainElement("edge-gateway")) + }) + E2EAfterEach(func() { + Expect(cluster.DismissCluster()).ToNot(HaveOccurred()) + }) + + // ProxySimpleRequests tests that basic HTTP requests are proxied to a service. + ProxySimpleRequests := func() { Eventually(func(g Gomega) { - host := net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080") - p := path.Join("test", url.PathEscape(GinkgoT().Name())) - target := fmt.Sprintf("http://%s/%s", host, p) + target := fmt.Sprintf("http://%s/%s", + net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080"), + path.Join("test", url.PathEscape(GinkgoT().Name())), + ) response, err := testutil.CollectResponse( cluster, "gateway-client", target, @@ -187,5 +192,56 @@ conf: g.Expect(response.Instance).To(Equal("universal")) g.Expect(response.Received.Headers["Host"]).To(ContainElement("example.kuma.io")) }, "30s", "1s").Should(Succeed()) + } + + Context("when mTLS is disabled", func() { + BeforeEach(func() { + DeployCluster(KumaUniversalDeployOpts...) + }) + + It("should proxy simple HTTP requests", ProxySimpleRequests) + }) + + Context("when mTLS is enabled", func() { + BeforeEach(func() { + mtls := WithMeshUpdate("default", func(mesh *mesh_proto.Mesh) *mesh_proto.Mesh { + mesh.Mtls = &mesh_proto.Mesh_Mtls{ + EnabledBackend: "builtin", + Backends: []*mesh_proto.CertificateAuthorityBackend{ + {Name: "builtin", Type: "builtin"}, + }, + } + return mesh + }) + + DeployCluster(append(KumaUniversalDeployOpts, mtls)...) + }) + + It("should proxy simple HTTP requests", ProxySimpleRequests) + + // In mTLS mode, only the presence of TrafficPermission rules allow services to receive + // traffic, so removing the permission should cause requests to fail. We use this to + // prove that mTLS is enabled + It("should fail without TrafficPermission", func() { + Expect( + cluster.GetKumactlOptions().KumactlDelete( + "traffic-permission", "allow-all-default", "default"), + ).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + target := fmt.Sprintf("http://%s/%s", + net.JoinHostPort(cluster.GetApp("gateway-proxy").GetIP(), "8080"), + path.Join("test", url.PathEscape(GinkgoT().Name())), + ) + + status, err := testutil.CollectFailure( + cluster, "gateway-client", target, + testutil.WithHeader("Host", "example.kuma.io"), + ) + + g.Expect(err).To(Succeed()) + g.Expect(status.ResponseCode).To(Equal(503)) + }, "30s", "1s").Should(Succeed()) + }) }) } diff --git a/test/e2e/trafficroute/testutil/collect.go b/test/e2e/trafficroute/testutil/collect.go index c90541c6db73..e06a15880a49 100644 --- a/test/e2e/trafficroute/testutil/collect.go +++ b/test/e2e/trafficroute/testutil/collect.go @@ -3,9 +3,11 @@ package testutil import ( "encoding/json" "fmt" + "os" "sync" "github.com/kballard/go-shellquote" + "github.com/pkg/errors" "github.com/kumahq/kuma/test/framework" "github.com/kumahq/kuma/test/server/types" @@ -71,6 +73,75 @@ func CollectResponse(cluster framework.Cluster, source, destination string, fn . return *response, nil } +// FailureResponse is the JSON output for a Curl command. Note that the available +// fields depend on the Curl version, which must be at least 7.70.0 for this feature. +// +// See https://curl.se/docs/manpage.html#-w. +type FailureResponse struct { + Errormsg string `json:"errormsg"` + Exitcode int `json:"exitcode"` + + ResponseCode int `json:"response_code"` + Method string `json:"method"` + Scheme string `json:"scheme"` + ContentType string `json:"content_type"` + URL string `json:"url"` + EffectiveURL string `json:"url_effective"` +} + +// CollectFailure runs Curl to fetch a URL that is expected to fail. The +// Curl JSON output is returned so the caller can inspect the failure to +// see whether it was what was expected. +func CollectFailure(cluster framework.Cluster, source, destination string, fn ...CollectResponsesOptsFn) (FailureResponse, error) { + opts := DefaultCollectResponsesOpts() + for _, f := range fn { + f(&opts) + } + + cmd := []string{ + "curl", + "--request", opts.Method, + "--max-time", "3", + "--silent", // Suppress human-readable errors. + "--write-out", "%{json}", // Write JSON result. Requires curl 7.70.0, April 2020. + // Silence output so that we don't try to parse it. A future refactor could try to address this + // by using "%{stderr}%{json}", but that needs a bit more investigation. + "--output", os.DevNull, + } + + for key, value := range opts.Headers { + cmd = append(cmd, "--header", shellquote.Join(fmt.Sprintf("%s: %s", key, value))) + } + + cmd = append(cmd, shellquote.Join(destination)) + stdout, _, err := cluster.Exec("", "", source, cmd...) + + // 1. If we fail to decode the JSON status, return the JSON error, + // but prefer the original error if we have it. + empty := FailureResponse{} + response := FailureResponse{} + if jsonErr := json.Unmarshal([]byte(stdout), &response); jsonErr != nil { + // Prefer the original error to a JSON decoding error. + if err == nil { + return response, jsonErr + } + } + + // 2. If there was no error response, we still prefer the original + // error, but fall back to reporting that the JSON is missing. + if response == empty { + if err != nil { + return response, err + } + + return response, errors.Errorf("empty JSON response from curl: %q", stdout) + } + + // 3. Finally, report the JSON status and no execution error + // since the JSON contains all the Curl error information. + return response, nil +} + func CollectResponses(cluster framework.Cluster, source, destination string, fn ...CollectResponsesOptsFn) ([]types.EchoResponse, error) { opts := DefaultCollectResponsesOpts() for _, f := range fn { diff --git a/test/framework/interface.go b/test/framework/interface.go index 095f9e8cbee4..26660f446999 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -6,6 +6,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/testing" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/config/core" ) @@ -44,6 +45,10 @@ type kumaDeploymentOptions struct { cpReplicas int hdsDisabled bool runPostgresMigration bool + + // Functions to apply to each mesh after the control plane + // is provisioned. + meshUpdateFuncs map[string][]func(*mesh_proto.Mesh) *mesh_proto.Mesh } func (k *kumaDeploymentOptions) apply(opts ...KumaDeploymentOption) { @@ -51,6 +56,7 @@ func (k *kumaDeploymentOptions) apply(opts ...KumaDeploymentOption) { k.isipv6 = IsIPv6() k.installationMode = KumactlInstallationMode k.env = map[string]string{} + k.meshUpdateFuncs = map[string][]func(*mesh_proto.Mesh) *mesh_proto.Mesh{} // Apply options. for _, o := range opts { @@ -269,6 +275,18 @@ func WithCtlOpt(name, value string) KumaDeploymentOption { }) } +type MeshUpdateFunc func(mesh *mesh_proto.Mesh) *mesh_proto.Mesh + +// WithMeshUpdate registers a function to update the specification +// for the named mesh. When the control plane implementation creates the +// mesh, it invokes the function and applies configuration changes to the +// mesh object. +func WithMeshUpdate(mesh string, u MeshUpdateFunc) KumaDeploymentOption { + return KumaOptionFunc(func(o *kumaDeploymentOptions) { + o.meshUpdateFuncs[mesh] = append(o.meshUpdateFuncs[mesh], u) + }) +} + // WithoutDataplane suppresses the automatic configuration of kuma-dp // in the application container. This is useful when the test requires a // container that is not bound to the mesh. diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go index a8ae3af04cfa..794f53673f25 100644 --- a/test/framework/k8s_cluster.go +++ b/test/framework/k8s_cluster.go @@ -24,12 +24,16 @@ import ( "go.uber.org/multierr" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/transport/spdy" "github.com/kumahq/kuma/pkg/config/core" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + resources_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s" + mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1" kuma_version "github.com/kumahq/kuma/pkg/version" ) @@ -540,11 +544,41 @@ func (c *K8sCluster) DeployKuma(mode core.CpMode, opt ...KumaDeploymentOption) e } } - err = c.controlplane.FinalizeAdd() - if err != nil { + if err := c.controlplane.FinalizeAdd(); err != nil { return err } + converter := resources_k8s.NewSimpleConverter() + for name, updateFuncs := range opts.meshUpdateFuncs { + for _, f := range updateFuncs { + Logf("applying update function to mesh %q", name) + err := c.controlplane.UpdateObject("mesh", name, + func(obj runtime.Object) runtime.Object { + mesh := core_mesh.NewMeshResource() + + // The kubectl updater should have already converted the Kubernetes object + // to a concrete type, so we can safely cast here. + if err := converter.ToCoreResource(obj.(*mesh_k8s.Mesh), mesh); err != nil { + panic(err.Error()) + } + + // Apply the conversion function. + mesh.Spec = f(mesh.Spec) + + // Convert back to a Kubernetes resource. + meshObj, err := converter.ToKubernetesObject(mesh) + if err != nil { + panic(err.Error()) + } + + return meshObj + }) + if err != nil { + return err + } + } + } + return nil } diff --git a/test/framework/k8s_controlplane.go b/test/framework/k8s_controlplane.go index 45be72e81356..57e8504e231d 100644 --- a/test/framework/k8s_controlplane.go +++ b/test/framework/k8s_controlplane.go @@ -1,10 +1,12 @@ package framework import ( + "bytes" "crypto/tls" "fmt" "net" "net/http" + "os" "strconv" http_helper "github.com/gruntwork-io/terratest/modules/http-helper" @@ -13,8 +15,13 @@ import ( "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/yaml" "github.com/kumahq/kuma/pkg/config/core" + bootstrap_k8s "github.com/kumahq/kuma/pkg/plugins/bootstrap/k8s" util_net "github.com/kumahq/kuma/pkg/util/net" ) @@ -282,3 +289,63 @@ func (c *K8sControlPlane) GenerateZoneIngressToken(zone string) (string, error) &tls.Config{}, ) } + +// UpdateObject fetches an object and updates it after the update function is applied to it. +func (c *K8sControlPlane) UpdateObject( + typeName string, + objectName string, + update func(object runtime.Object) runtime.Object, +) error { + scheme, err := bootstrap_k8s.NewScheme() + if err != nil { + return err + } + + out, err := k8s.RunKubectlAndGetOutputE(c.t, c.GetKubectlOptions(), "get", typeName, objectName, "-o", "yaml") + if err != nil { + return err + } + + decoder := yaml.NewYAMLToJSONDecoder(bytes.NewReader([]byte(out))) + into := map[string]interface{}{} + + if err := decoder.Decode(&into); err != nil { + return err + } + + u := unstructured.Unstructured{Object: into} + obj, err := scheme.New(u.GroupVersionKind()) + if err != nil { + return err + } + + if err := scheme.Convert(u, obj, nil); err != nil { + return nil + } + + obj = update(obj) + + codecs := serializer.NewCodecFactory(scheme) + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeYAML) + if !ok { + return errors.Errorf("no serializer for %q", runtime.ContentTypeYAML) + } + + encoder := codecs.EncoderForVersion(info.Serializer, obj.GetObjectKind().GroupVersionKind().GroupVersion()) + yaml, err := runtime.Encode(encoder, obj) + if err != nil { + return err + } + + KubectlReplaceFromStringE := func(t testing.TestingT, options *k8s.KubectlOptions, configData string) error { + tmpfile, err := k8s.StoreConfigToTempFileE(t, configData) + if err != nil { + return err + } + defer os.Remove(tmpfile) + + return k8s.RunKubectlE(t, options, "replace", "-f", tmpfile) + } + + return KubectlReplaceFromStringE(c.t, c.GetKubectlOptions(), string(yaml)) +} diff --git a/test/framework/kumactl.go b/test/framework/kumactl.go index 38c6299844c9..4ef1eb176349 100644 --- a/test/framework/kumactl.go +++ b/test/framework/kumactl.go @@ -14,6 +14,8 @@ import ( "github.com/pkg/errors" "github.com/kumahq/kuma/pkg/config/core" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" + "github.com/kumahq/kuma/pkg/core/resources/model/rest" ) type KumactlOptions struct { @@ -45,44 +47,44 @@ func NewKumactlOptions(t testing.TestingT, cpname string, verbose bool) (*Kumact }, nil } -func (o *KumactlOptions) RunKumactl(args ...string) error { - out, err := o.RunKumactlAndGetOutput(args...) +func (k *KumactlOptions) RunKumactl(args ...string) error { + out, err := k.RunKumactlAndGetOutput(args...) if err != nil { return errors.Wrapf(err, out) } return nil } -func (o *KumactlOptions) RunKumactlAndGetOutput(args ...string) (string, error) { - return o.RunKumactlAndGetOutputV(o.Verbose, args...) +func (k *KumactlOptions) RunKumactlAndGetOutput(args ...string) (string, error) { + return k.RunKumactlAndGetOutputV(k.Verbose, args...) } -func (o *KumactlOptions) RunKumactlAndGetOutputV(verbose bool, args ...string) (string, error) { +func (k *KumactlOptions) RunKumactlAndGetOutputV(verbose bool, args ...string) (string, error) { cmdArgs := []string{} - if o.ConfigPath != "" { - cmdArgs = append(cmdArgs, "--config-file", o.ConfigPath) + if k.ConfigPath != "" { + cmdArgs = append(cmdArgs, "--config-file", k.ConfigPath) } cmdArgs = append(cmdArgs, args...) command := shell.Command{ - Command: o.Kumactl, + Command: k.Kumactl, Args: cmdArgs, - Env: o.Env, + Env: k.Env, } if !verbose { command.Logger = logger.Discard } - return shell.RunCommandAndGetStdOutE(o.t, command) + return shell.RunCommandAndGetStdOutE(k.t, command) } -func (o *KumactlOptions) KumactlDelete(kumatype, name, mesh string) error { - return o.RunKumactl("delete", kumatype, name, "--mesh", mesh) +func (k *KumactlOptions) KumactlDelete(kumatype, name, mesh string) error { + return k.RunKumactl("delete", kumatype, name, "--mesh", mesh) } -func (o *KumactlOptions) KumactlList(kumatype, mesh string) ([]string, error) { - out, err := o.RunKumactlAndGetOutput("get", kumatype, "--mesh", mesh, "-o", "json") +func (k *KumactlOptions) KumactlList(kumatype, mesh string) ([]string, error) { + out, err := k.RunKumactlAndGetOutput("get", kumatype, "--mesh", mesh, "-o", "json") if err != nil { return nil, err } @@ -106,19 +108,19 @@ func (o *KumactlOptions) KumactlList(kumatype, mesh string) ([]string, error) { return items, nil } -func (o *KumactlOptions) KumactlApply(configPath string) error { - return o.RunKumactl("apply", "-f", configPath) +func (k *KumactlOptions) KumactlApply(configPath string) error { + return k.RunKumactl("apply", "-f", configPath) } -func (o *KumactlOptions) KumactlApplyFromString(configData string) error { - tmpfile, err := storeConfigToTempFile(o.t.Name(), configData) +func (k *KumactlOptions) KumactlApplyFromString(configData string) error { + tmpfile, err := storeConfigToTempFile(k.t.Name(), configData) if err != nil { return err } defer os.Remove(tmpfile) - return o.KumactlApply(tmpfile) + return k.KumactlApply(tmpfile) } func storeConfigToTempFile(name string, configData string) (string, error) { @@ -135,7 +137,7 @@ func storeConfigToTempFile(name string, configData string) (string, error) { return tmpfile.Name(), err } -func (o *KumactlOptions) KumactlInstallCP(mode string, args ...string) (string, error) { +func (k *KumactlOptions) KumactlInstallCP(mode string, args ...string) (string, error) { cmd := []string{ "install", "control-plane", } @@ -143,7 +145,7 @@ func (o *KumactlOptions) KumactlInstallCP(mode string, args ...string) (string, cmd = append(cmd, "--mode", mode) switch mode { case core.Zone: - cmd = append(cmd, "--zone", o.CPName) + cmd = append(cmd, "--zone", k.CPName) fallthrough case core.Global: if !UseLoadBalancer() { @@ -153,31 +155,31 @@ func (o *KumactlOptions) KumactlInstallCP(mode string, args ...string) (string, cmd = append(cmd, args...) - return o.RunKumactlAndGetOutputV( + return k.RunKumactlAndGetOutputV( false, // silence the log output of Install cmd...) } -func (o *KumactlOptions) KumactlInstallDNS(args ...string) (string, error) { +func (k *KumactlOptions) KumactlInstallDNS(args ...string) (string, error) { args = append([]string{"install", "dns"}, args...) - return o.RunKumactlAndGetOutputV( + return k.RunKumactlAndGetOutputV( false, // silence the log output of Install args...) } -func (o *KumactlOptions) KumactlInstallMetrics() (string, error) { - return o.RunKumactlAndGetOutput("install", "metrics") +func (k *KumactlOptions) KumactlInstallMetrics() (string, error) { + return k.RunKumactlAndGetOutput("install", "metrics") } -func (o *KumactlOptions) KumactlInstallTracing() (string, error) { - return o.RunKumactlAndGetOutput("install", "tracing") +func (k *KumactlOptions) KumactlInstallTracing() (string, error) { + return k.RunKumactlAndGetOutput("install", "tracing") } -func (o *KumactlOptions) KumactlConfigControlPlanesAdd(name, address string) error { - _, err := retry.DoWithRetryE(o.t, "kumactl config control-planes add", DefaultRetries, DefaultTimeout, +func (k *KumactlOptions) KumactlConfigControlPlanesAdd(name, address string) error { + _, err := retry.DoWithRetryE(k.t, "kumactl config control-planes add", DefaultRetries, DefaultTimeout, func() (string, error) { - err := o.RunKumactl( + err := k.RunKumactl( "config", "control-planes", "add", "--overwrite", "--name", name, @@ -192,3 +194,28 @@ func (o *KumactlOptions) KumactlConfigControlPlanesAdd(name, address string) err return err } + +// KumactlUpdateObject fetches an object and updates it after the update function is applied to it. +func (k *KumactlOptions) KumactlUpdateObject( + typeName string, + objectName string, + update func(core_model.Resource) core_model.Resource, +) error { + out, err := k.RunKumactlAndGetOutput("get", typeName, objectName, "-o", "yaml") + if err != nil { + return errors.Wrapf(err, "failed to get %q object %q", typeName, objectName) + } + + resource, err := rest.UnmarshallToCore([]byte(out)) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal %q object %q", typeName, objectName) + } + + updated := rest.NewFromModel(update(resource)) + json, err := updated.MarshalJSON() + if err != nil { + return errors.Wrapf(err, "failed to marshal JSON for %q object %q", typeName, objectName) + } + + return k.KumactlApplyFromString(string(json)) +} diff --git a/test/framework/universal_app.go b/test/framework/universal_app.go index 6c08c58f7ea9..b2d7576d4f95 100644 --- a/test/framework/universal_app.go +++ b/test/framework/universal_app.go @@ -246,7 +246,7 @@ func NewUniversalApp(t testing.TestingT, clusterName, dpName string, mode AppMod return "Success", nil }) - fmt.Printf("Node IP %s\n", app.ip) + Logf("Node IP %s", app.ip) return app, nil } @@ -509,12 +509,12 @@ func NewSshApp(verbose bool, port string, env []string, args []string) *SshApp { } func (s *SshApp) Run() error { - fmt.Printf("Running %v\n", s.cmd) + Logf("Running %v", s.cmd) return s.cmd.Run() } func (s *SshApp) Start() error { - fmt.Printf("Starting %v\n", s.cmd) + Logf("Starting %v", s.cmd) return s.cmd.Start() } diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go index ea012e525df8..f3453aeced3e 100644 --- a/test/framework/universal_cluster.go +++ b/test/framework/universal_cluster.go @@ -13,6 +13,8 @@ import ( "go.uber.org/multierr" "github.com/kumahq/kuma/pkg/config/core" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/util/template" ) @@ -139,6 +141,21 @@ func (c *UniversalCluster) DeployKuma(mode core.CpMode, opt ...KumaDeploymentOpt return err } + for name, updateFuncs := range opts.meshUpdateFuncs { + for _, f := range updateFuncs { + Logf("applying update function to mesh %q", name) + err := c.controlplane.kumactl.KumactlUpdateObject("mesh", name, + func(resource core_model.Resource) core_model.Resource { + mesh := resource.(*core_mesh.MeshResource) + mesh.Spec = f(mesh.Spec) + return mesh + }) + if err != nil { + return err + } + } + } + c.apps[AppModeCP] = app return nil @@ -215,7 +232,7 @@ func (c *UniversalCluster) DeployApp(opt ...AppDeploymentOption) error { caps = append(caps, "NET_ADMIN", "NET_RAW") } - Logf("IPV6 is %v\n", opts.isipv6) + Logf("IPV6 is %v", opts.isipv6) app, err := NewUniversalApp(c.t, c.name, opts.name, AppMode(appname), opts.isipv6, *opts.verbose, caps) if err != nil { diff --git a/test/framework/universal_controlplane.go b/test/framework/universal_controlplane.go index 599674894b66..6963fc68dd76 100644 --- a/test/framework/universal_controlplane.go +++ b/test/framework/universal_controlplane.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/kumahq/kuma/pkg/config/core" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" ) type UniversalControlPlane struct { @@ -104,3 +105,11 @@ func (c *UniversalControlPlane) GenerateZoneIngressToken(zone string) (string, e return sshApp.Out(), nil }) } + +func (c *UniversalControlPlane) UpdateObject( + typeName string, + objectName string, + update func(object core_model.Resource) core_model.Resource, +) error { + return c.kumactl.KumactlUpdateObject(typeName, objectName, update) +}