From 08691ad8c7d5f298d11088fa42772b0e13bfc682 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Tue, 22 Oct 2019 16:00:23 +0200 Subject: [PATCH] feat(*) path file is passed by metadata --- Makefile | 4 +- app/kuma-dp/cmd/run.go | 6 ++ app/kuma-dp/pkg/config/config_suite_test.go | 13 +++ app/kuma-dp/pkg/config/validate.go | 20 ++++ app/kuma-dp/pkg/config/validate_test.go | 57 ++++++++++++ .../pkg/dataplane/envoy/remote_bootstrap.go | 3 +- .../dataplane/envoy/remote_bootstrap_test.go | 4 +- app/kuma-injector/pkg/injector/injector.go | 4 + .../injector/testdata/inject.01.golden.yaml | 2 + .../injector/testdata/inject.02.golden.yaml | 2 + .../injector/testdata/inject.03.golden.yaml | 2 + .../injector/testdata/inject.04.golden.yaml | 2 + .../injector/testdata/inject.05.golden.yaml | 2 + app/kumactl/pkg/cmd/root_context.go | 3 +- app/kumactl/pkg/config/io.go | 10 +- pkg/config/app/kuma-dp/config.go | 3 +- pkg/config/app/kuma-dp/config_test.go | 2 + pkg/core/xds/metadata.go | 18 ++++ pkg/core/xds/metadata_test.go | 44 +++++++++ pkg/core/xds/types.go | 1 + pkg/util/files/files.go | 16 ++++ pkg/xds/bootstrap/generator.go | 1 + pkg/xds/bootstrap/generator_test.go | 43 ++++++++- pkg/xds/bootstrap/server_test.go | 4 +- pkg/xds/bootstrap/template.go | 5 + .../testdata/bootstrap.overridden.golden.yaml | 2 + ....custom-config-minimal-request.golden.yaml | 50 ++++++++++ .../generator.custom-config.golden.yaml | 4 +- ...default-config-minimal-request.golden.yaml | 44 +++++++++ .../generator.default-config.golden.yaml | 8 ++ pkg/xds/bootstrap/types/bootstrap_request.go | 7 +- pkg/xds/context/context.go | 9 -- pkg/xds/envoy/envoy.go | 29 +++--- pkg/xds/envoy/envoy_test.go | 92 +++++-------------- .../generator/inbound_proxy_generator_test.go | 1 + .../outbound_proxy_generator_test.go | 3 +- pkg/xds/generator/proxy_template.go | 4 +- .../proxy_template_profile_source_test.go | 1 + pkg/xds/generator/proxy_template_test.go | 1 + pkg/xds/server/components.go | 10 +- pkg/xds/server/components_test.go | 23 +---- pkg/xds/server/dataplane_metadata_tracker.go | 71 ++++++++++++++ .../server/dataplane_metadata_tracker_test.go | 73 +++++++++++++++ pkg/xds/server/snapshot_generator_test.go | 3 +- pkg/xds/sync/dataplane_sync_tracker.go | 4 +- pkg/xds/sync/dataplane_sync_tracker_test.go | 2 +- 46 files changed, 565 insertions(+), 147 deletions(-) create mode 100644 app/kuma-dp/pkg/config/config_suite_test.go create mode 100644 app/kuma-dp/pkg/config/validate.go create mode 100644 app/kuma-dp/pkg/config/validate_test.go create mode 100644 pkg/core/xds/metadata.go create mode 100644 pkg/core/xds/metadata_test.go create mode 100644 pkg/util/files/files.go create mode 100644 pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml create mode 100644 pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml create mode 100644 pkg/xds/server/dataplane_metadata_tracker.go create mode 100644 pkg/xds/server/dataplane_metadata_tracker_test.go diff --git a/Makefile b/Makefile index fa7cfdc35d6f..9e56053418ea 100644 --- a/Makefile +++ b/Makefile @@ -441,11 +441,13 @@ run/example/envoy/k8s: run/example/envoy run/example/envoy/universal: run/example/envoy -run/example/envoy: build/kuma-dp ## Dev: Run Envoy configured against local Control Plane +run/example/envoy: build/kuma-dp build/kumactl ## Dev: Run Envoy configured against local Control Plane + ${BUILD_ARTIFACTS_DIR}/kumactl/kumactl generate dataplane-token --name=$(EXAMPLE_DATAPLANE_NAME) --mesh=$(EXAMPLE_DATAPLANE_MESH) > /tmp/kuma-dp-$(EXAMPLE_DATAPLANE_NAME)-$(EXAMPLE_DATAPLANE_MESH)-token KUMA_CONTROL_PLANE_BOOTSTRAP_SERVER_URL=http://localhost:5682 \ KUMA_DATAPLANE_MESH=$(EXAMPLE_DATAPLANE_MESH) \ KUMA_DATAPLANE_NAME=$(EXAMPLE_DATAPLANE_NAME) \ KUMA_DATAPLANE_ADMIN_PORT=$(ENVOY_ADMIN_PORT) \ + KUMA_DATAPLANE_RUNTIME_TOKEN_PATH=/tmp/kuma-dp-$(EXAMPLE_DATAPLANE_NAME)-$(EXAMPLE_DATAPLANE_MESH)-token \ ${BUILD_ARTIFACTS_DIR}/kuma-dp/kuma-dp run --log-level=debug config_dump/example/envoy: ## Dev: Dump effective configuration of example Envoy diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 48d1aaab9a6c..7e61f7d81563 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + kumadp_config "github.com/Kong/kuma/app/kuma-dp/pkg/config" "github.com/Kong/kuma/app/kuma-dp/pkg/dataplane/accesslogs" "github.com/Kong/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/Kong/kuma/pkg/config" @@ -41,6 +42,10 @@ func newRunCmd() *cobra.Command { return err } + if err := kumadp_config.ValidateTokenPath(cfg.DataplaneRuntime.TokenPath); err != nil { + return err + } + if cfg.DataplaneRuntime.ConfigDir == "" { tmpDir, err := ioutil.TempDir("", "kuma-dp-") if err != nil { @@ -106,5 +111,6 @@ func newRunCmd() *cobra.Command { cmd.PersistentFlags().StringVar(&cfg.ControlPlane.BootstrapServer.URL, "cp-address", cfg.ControlPlane.BootstrapServer.URL, "Mesh that Dataplane belongs to") cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.BinaryPath, "binary-path", cfg.DataplaneRuntime.BinaryPath, "Binary path of Envoy executable") cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.ConfigDir, "config-dir", cfg.DataplaneRuntime.ConfigDir, "Directory in which Envoy config will be generated") + cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.TokenPath, "dataplane-token", cfg.DataplaneRuntime.TokenPath, "Path to a file with dataplane token (use 'kumactl generate dataplane-token' to get one)") return cmd } diff --git a/app/kuma-dp/pkg/config/config_suite_test.go b/app/kuma-dp/pkg/config/config_suite_test.go new file mode 100644 index 000000000000..ddc35f3ff784 --- /dev/null +++ b/app/kuma-dp/pkg/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kuma DP Config Suite") +} diff --git a/app/kuma-dp/pkg/config/validate.go b/app/kuma-dp/pkg/config/validate.go new file mode 100644 index 000000000000..9079ac18e778 --- /dev/null +++ b/app/kuma-dp/pkg/config/validate.go @@ -0,0 +1,20 @@ +package config + +import ( + util_files "github.com/Kong/kuma/pkg/util/files" + "github.com/pkg/errors" +) + +func ValidateTokenPath(path string) error { + if path == "" { + return nil + } + empty, err := util_files.FileEmpty(path) + if err != nil { + return errors.Wrap(err, "could not read file") + } + if empty { + return errors.Errorf("token under file %s is empty", path) + } + return nil +} diff --git a/app/kuma-dp/pkg/config/validate_test.go b/app/kuma-dp/pkg/config/validate_test.go new file mode 100644 index 000000000000..9a6ead71cc71 --- /dev/null +++ b/app/kuma-dp/pkg/config/validate_test.go @@ -0,0 +1,57 @@ +package config_test + +import ( + "fmt" + "github.com/Kong/kuma/app/kuma-dp/pkg/config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "os" +) + +var _ = Describe("ValidateTokenPath", func() { + + var tokenFile *os.File + + BeforeEach(func() { + tf, err := ioutil.TempFile("", "") + Expect(err).ToNot(HaveOccurred()) + tokenFile = tf + }) + + It("should pass validation for empty path", func() { + // when + err := config.ValidateTokenPath("") + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should pass validation for empty path", func() { + // given + _, err := tokenFile.WriteString("sampletoken") + Expect(err).ToNot(HaveOccurred()) + + // when + err = config.ValidateTokenPath("") + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should fail for non existing file", func() { + // when + err := config.ValidateTokenPath("nonexistingfile") + + // then + Expect(err).To(MatchError("could not read file: stat nonexistingfile: no such file or directory")) + }) + + It("should fail for empty file", func() { + // when + err := config.ValidateTokenPath(tokenFile.Name()) + + // then + Expect(err).To(MatchError(fmt.Sprintf("token under file %s is empty", tokenFile.Name()))) + }) +}) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index 3dfba7d62225..85f0cbfcc2f6 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -30,7 +30,8 @@ func (b *remoteBootstrap) Generate(cfg kuma_dp.Config) (proto.Message, error) { Name: cfg.Dataplane.Name, // if not set in config, the 0 will be sent which will result in providing default admin port // that is set in the control plane bootstrap params - AdminPort: cfg.Dataplane.AdminPort, + AdminPort: cfg.Dataplane.AdminPort, + DataplaneTokenPath: cfg.DataplaneRuntime.TokenPath, } jsonBytes, err := json.Marshal(request) if err != nil { diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index 07caec90b7d9..bff3246780f3 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -28,7 +28,8 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", - "adminPort": 4321 + "adminPort": 4321, + "dataplaneTokenPath": "/tmp/token" } `)) @@ -47,6 +48,7 @@ var _ = Describe("Remote Bootstrap", func() { cfg.Dataplane.Mesh = "demo" cfg.Dataplane.Name = "sample" cfg.Dataplane.AdminPort = 4321 + cfg.DataplaneRuntime.TokenPath = "/tmp/token" cfg.ControlPlane.BootstrapServer.URL = fmt.Sprintf("http://localhost:%d", port) // when diff --git a/app/kuma-injector/pkg/injector/injector.go b/app/kuma-injector/pkg/injector/injector.go index 1583ede7a957..e225e4a57a46 100644 --- a/app/kuma-injector/pkg/injector/injector.go +++ b/app/kuma-injector/pkg/injector/injector.go @@ -113,6 +113,10 @@ func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod) kube_core.Contain Name: "KUMA_DATAPLANE_DRAIN_TIME", Value: fmt.Sprintf("%s", i.cfg.SidecarContainer.DrainTime), }, + { + Name: "KUMA_DATAPLANE_RUNTIME_TOKEN_PATH", + Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, }, SecurityContext: &kube_core.SecurityContext{ RunAsUser: &i.cfg.SidecarContainer.UID, diff --git a/app/kuma-injector/pkg/injector/testdata/inject.01.golden.yaml b/app/kuma-injector/pkg/injector/testdata/inject.01.golden.yaml index 4faec9d16bb0..b67fff365594 100644 --- a/app/kuma-injector/pkg/injector/testdata/inject.01.golden.yaml +++ b/app/kuma-injector/pkg/injector/testdata/inject.01.golden.yaml @@ -48,6 +48,8 @@ spec: value: "9901" - name: KUMA_DATAPLANE_DRAIN_TIME value: 31s + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token image: kuma/kuma-sidecar:latest imagePullPolicy: IfNotPresent livenessProbe: diff --git a/app/kuma-injector/pkg/injector/testdata/inject.02.golden.yaml b/app/kuma-injector/pkg/injector/testdata/inject.02.golden.yaml index aa1dbcbbbd10..4e8b5050bfad 100644 --- a/app/kuma-injector/pkg/injector/testdata/inject.02.golden.yaml +++ b/app/kuma-injector/pkg/injector/testdata/inject.02.golden.yaml @@ -49,6 +49,8 @@ spec: value: "9901" - name: KUMA_DATAPLANE_DRAIN_TIME value: 31s + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token image: kuma/kuma-sidecar:latest imagePullPolicy: IfNotPresent livenessProbe: diff --git a/app/kuma-injector/pkg/injector/testdata/inject.03.golden.yaml b/app/kuma-injector/pkg/injector/testdata/inject.03.golden.yaml index 19574e0144f6..343ff9bb36d4 100644 --- a/app/kuma-injector/pkg/injector/testdata/inject.03.golden.yaml +++ b/app/kuma-injector/pkg/injector/testdata/inject.03.golden.yaml @@ -108,6 +108,8 @@ spec: value: "9901" - name: KUMA_DATAPLANE_DRAIN_TIME value: 31s + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token image: kuma/kuma-sidecar:latest imagePullPolicy: IfNotPresent livenessProbe: diff --git a/app/kuma-injector/pkg/injector/testdata/inject.04.golden.yaml b/app/kuma-injector/pkg/injector/testdata/inject.04.golden.yaml index a43271829fcd..7254d4e8932c 100644 --- a/app/kuma-injector/pkg/injector/testdata/inject.04.golden.yaml +++ b/app/kuma-injector/pkg/injector/testdata/inject.04.golden.yaml @@ -48,6 +48,8 @@ spec: value: "9901" - name: KUMA_DATAPLANE_DRAIN_TIME value: 31s + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token image: kuma/kuma-sidecar:latest imagePullPolicy: IfNotPresent livenessProbe: diff --git a/app/kuma-injector/pkg/injector/testdata/inject.05.golden.yaml b/app/kuma-injector/pkg/injector/testdata/inject.05.golden.yaml index 8419a3e4dd7c..f9db2f5857cb 100644 --- a/app/kuma-injector/pkg/injector/testdata/inject.05.golden.yaml +++ b/app/kuma-injector/pkg/injector/testdata/inject.05.golden.yaml @@ -45,6 +45,8 @@ spec: value: "9901" - name: KUMA_DATAPLANE_DRAIN_TIME value: 31s + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token image: kuma/kuma-sidecar:latest imagePullPolicy: IfNotPresent livenessProbe: diff --git a/app/kumactl/pkg/cmd/root_context.go b/app/kumactl/pkg/cmd/root_context.go index 93ab682f2298..d74b12a72cac 100644 --- a/app/kumactl/pkg/cmd/root_context.go +++ b/app/kumactl/pkg/cmd/root_context.go @@ -11,6 +11,7 @@ import ( config_proto "github.com/Kong/kuma/pkg/config/app/kumactl/v1alpha1" core_model "github.com/Kong/kuma/pkg/core/resources/model" core_store "github.com/Kong/kuma/pkg/core/resources/store" + util_files "github.com/Kong/kuma/pkg/util/files" "github.com/pkg/errors" kumactl_resources "github.com/Kong/kuma/app/kumactl/pkg/resources" @@ -130,5 +131,5 @@ func (rc *RootContext) CurrentDataplaneTokenClient() (tokens.DataplaneTokenClien } func (rc *RootContext) IsFirstTimeUsage() bool { - return rc.Args.ConfigFile == "" && !config.FileExists(config.DefaultConfigFile) + return rc.Args.ConfigFile == "" && !util_files.FileExists(config.DefaultConfigFile) } diff --git a/app/kumactl/pkg/config/io.go b/app/kumactl/pkg/config/io.go index 1a356b9a3c5b..f3122e7c94ab 100644 --- a/app/kumactl/pkg/config/io.go +++ b/app/kumactl/pkg/config/io.go @@ -6,6 +6,7 @@ import ( "path/filepath" config_proto "github.com/Kong/kuma/pkg/config/app/kumactl/v1alpha1" + util_files "github.com/Kong/kuma/pkg/util/files" util_proto "github.com/Kong/kuma/pkg/util/proto" "github.com/pkg/errors" ) @@ -15,13 +16,13 @@ var DefaultConfigFile = filepath.Join(os.Getenv("HOME"), ".kumactl", "config") func Load(file string, cfg *config_proto.Configuration) error { configFile := DefaultConfigFile if file != "" { - if FileExists(file) { + if util_files.FileExists(file) { configFile = file } else { return errors.Errorf("Failed to access configuration file %q", file) } } - if FileExists(configFile) { + if util_files.FileExists(configFile) { if contents, err := ioutil.ReadFile(configFile); err != nil { return errors.Wrapf(err, "Failed to read configuration from file %q", configFile) } else if err := util_proto.FromYAML(contents, cfg); err != nil { @@ -57,8 +58,3 @@ func Save(file string, cfg *config_proto.Configuration) error { } return nil } - -func FileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index a303033eae61..907893196959 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -5,7 +5,6 @@ import ( "time" "github.com/Kong/kuma/pkg/config" - "github.com/pkg/errors" "go.uber.org/multierr" ) @@ -69,6 +68,8 @@ type DataplaneRuntime struct { BinaryPath string `yaml:"binaryPath,omitempty" envconfig:"kuma_dataplane_runtime_binary_path"` // Dir to store auto-generated Envoy bootstrap config in. ConfigDir string `yaml:"configDir,omitempty" envconfig:"kuma_dataplane_runtime_config_dir"` + // Path to a file with dataplane token (use 'kumactl generate dataplane-token' to get one) + TokenPath string `yaml:"dataplaneTokenPath,omitempty" envconfig:"kuma_dataplane_runtime_token_path"` } var _ config.Config = &Config{} diff --git a/pkg/config/app/kuma-dp/config_test.go b/pkg/config/app/kuma-dp/config_test.go index d3e71345d466..7679afd191b6 100644 --- a/pkg/config/app/kuma-dp/config_test.go +++ b/pkg/config/app/kuma-dp/config_test.go @@ -57,6 +57,7 @@ var _ = Describe("Config", func() { "KUMA_DATAPLANE_DRAIN_TIME": "60s", "KUMA_DATAPLANE_RUNTIME_BINARY_PATH": "envoy.sh", "KUMA_DATAPLANE_RUNTIME_CONFIG_DIR": "/var/run/envoy", + "KUMA_DATAPLANE_RUNTIME_TOKEN_PATH": "/tmp/token", } for key, value := range env { os.Setenv(key, value) @@ -79,6 +80,7 @@ var _ = Describe("Config", func() { Expect(cfg.Dataplane.DrainTime).To(Equal(60 * time.Second)) Expect(cfg.DataplaneRuntime.BinaryPath).To(Equal("envoy.sh")) Expect(cfg.DataplaneRuntime.ConfigDir).To(Equal("/var/run/envoy")) + Expect(cfg.DataplaneRuntime.TokenPath).To(Equal("/tmp/token")) }) }) diff --git a/pkg/core/xds/metadata.go b/pkg/core/xds/metadata.go new file mode 100644 index 000000000000..1ccf0208e730 --- /dev/null +++ b/pkg/core/xds/metadata.go @@ -0,0 +1,18 @@ +package xds + +import "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + +type DataplaneMetadata struct { + DataplaneTokenPath string +} + +func DataplaneMetadataFromNode(node *core.Node) *DataplaneMetadata { + metadata := DataplaneMetadata{} + if node.Metadata == nil { + return &metadata + } + if field := node.Metadata.Fields["dataplaneTokenPath"]; field != nil { + metadata.DataplaneTokenPath = field.GetStringValue() + } + return &metadata +} diff --git a/pkg/core/xds/metadata_test.go b/pkg/core/xds/metadata_test.go new file mode 100644 index 000000000000..7a332f100873 --- /dev/null +++ b/pkg/core/xds/metadata_test.go @@ -0,0 +1,44 @@ +package xds_test + +import ( + "github.com/Kong/kuma/pkg/core/xds" + "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + "github.com/gogo/protobuf/types" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" +) + +type testCase struct { + node core.Node + expected xds.DataplaneMetadata +} + +var _ = DescribeTable("DataplaneMetadataFromNode", + func(given testCase) { + // when + metadata := xds.DataplaneMetadataFromNode(&given.node) + + // then + Expect(*metadata).To(Equal(given.expected)) + }, + Entry("should parse metadata from empty node", testCase{ + node: core.Node{}, + expected: xds.DataplaneMetadata{}, + }), + Entry("should parse metadata", testCase{ + node: core.Node{ + Metadata: &types.Struct{ + Fields: map[string]*types.Value{ + "dataplaneTokenPath": &types.Value{ + Kind: &types.Value_StringValue{ + StringValue: "/tmp/token", + }, + }, + }, + }, + }, + expected: xds.DataplaneMetadata{ + DataplaneTokenPath: "/tmp/token", + }, + }), +) diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go index 4d43b2e3c33f..be48aa09cd47 100644 --- a/pkg/core/xds/types.go +++ b/pkg/core/xds/types.go @@ -31,6 +31,7 @@ type Proxy struct { TrafficPermissions permissions.MatchedPermissions Logs *logs.MatchedLogs OutboundTargets map[string][]net.SRV + Metadata *DataplaneMetadata } func BuildProxyId(mesh, name string, more ...string) (*ProxyId, error) { diff --git a/pkg/util/files/files.go b/pkg/util/files/files.go new file mode 100644 index 000000000000..c91b62e441c1 --- /dev/null +++ b/pkg/util/files/files.go @@ -0,0 +1,16 @@ +package files + +import "os" + +func FileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func FileEmpty(path string) (bool, error) { + file, err := os.Stat(path) + if err != nil { + return true, err + } + return file.Size() == 0, nil +} diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 8136fe98f6f8..373984052e83 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -67,6 +67,7 @@ func (b *bootstrapGenerator) GenerateFor(proxyId xds.ProxyId, dataplane *mesh.Da XdsPort: b.config.XdsPort, XdsConnectTimeout: b.config.XdsConnectTimeout, AccessLogPipe: accessLogPipe, + DataplaneTokenPath: request.DataplaneTokenPath, } log.WithValues("params", params).Info("Generating bootstrap config") return b.ConfigForParameters(params) diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index 03ccd8612222..9c5bc9e1ef8f 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -60,6 +60,7 @@ var _ = Describe("bootstrapGenerator", func() { type testCase struct { config *bootstrap_config.BootstrapParamsConfig + request types.BootstrapRequest expectedConfigFile string } DescribeTable("should generate bootstrap configuration", @@ -68,10 +69,7 @@ var _ = Describe("bootstrapGenerator", func() { generator := NewDefaultBootstrapGenerator(resManager, given.config) // when - bootstrapConfig, err := generator.Generate(context.Background(), types.BootstrapRequest{ - Mesh: "mesh", - Name: "name.namespace", - }) + bootstrapConfig, err := generator.Generate(context.Background(), given.request) // then Expect(err).ToNot(HaveOccurred()) @@ -88,10 +86,39 @@ var _ = Describe("bootstrapGenerator", func() { // expect Expect(actual).To(MatchYAML(expected)) }, + Entry("default config with minimal request", testCase{ + config: bootstrap_config.DefaultBootstrapParamsConfig(), + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + }, + expectedConfigFile: "generator.default-config-minimal-request.golden.yaml", + }), Entry("default config", testCase{ - config: bootstrap_config.DefaultBootstrapParamsConfig(), + config: bootstrap_config.DefaultBootstrapParamsConfig(), + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + AdminPort: 1234, + DataplaneTokenPath: "/tmp/token", + }, expectedConfigFile: "generator.default-config.golden.yaml", }), + Entry("custom config with minimal request", testCase{ + config: &bootstrap_config.BootstrapParamsConfig{ + AdminAddress: "192.168.0.1", // by default, Envoy Admin interface should listen on loopback address + AdminPort: 9902, // by default, turn off Admin interface of Envoy + AdminAccessLogPath: "/var/log", + XdsHost: "kuma-control-plane.internal", + XdsPort: 15678, + XdsConnectTimeout: 2 * time.Second, + }, + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + }, + expectedConfigFile: "generator.custom-config-minimal-request.golden.yaml", + }), Entry("custom config", testCase{ config: &bootstrap_config.BootstrapParamsConfig{ AdminAddress: "192.168.0.1", // by default, Envoy Admin interface should listen on loopback address @@ -101,6 +128,12 @@ var _ = Describe("bootstrapGenerator", func() { XdsPort: 15678, XdsConnectTimeout: 2 * time.Second, }, + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + AdminPort: 1234, + DataplaneTokenPath: "/tmp/token", + }, expectedConfigFile: "generator.custom-config.golden.yaml", }), ) diff --git a/pkg/xds/bootstrap/server_test.go b/pkg/xds/bootstrap/server_test.go index 3eb6e5f7fd08..8162bd5263b2 100644 --- a/pkg/xds/bootstrap/server_test.go +++ b/pkg/xds/bootstrap/server_test.go @@ -111,8 +111,8 @@ var _ = Describe("Bootstrap Server", func() { body: `{ "mesh": "default", "name": "dp-1.default" }`, expectedConfigFile: "bootstrap.k8s.golden.yaml", }), - Entry("overridden admin port", testCase{ - body: `{ "mesh": "default", "name": "dp-1.default", "adminPort": 1234 }`, + Entry("full data provided", testCase{ + body: `{ "mesh": "default", "name": "dp-1.default", "adminPort": 1234, "dataplaneTokenPath": "/tmp/token" }`, expectedConfigFile: "bootstrap.overridden.golden.yaml", }), ) diff --git a/pkg/xds/bootstrap/template.go b/pkg/xds/bootstrap/template.go index 0beb58f9b830..b4d3971be72f 100644 --- a/pkg/xds/bootstrap/template.go +++ b/pkg/xds/bootstrap/template.go @@ -12,12 +12,17 @@ type configParameters struct { XdsPort uint32 XdsConnectTimeout time.Duration AccessLogPipe string + DataplaneTokenPath string } const configTemplate string = ` node: id: {{.Id}} cluster: {{.Service}} + metadata: +{{if .DataplaneTokenPath}} + dataplaneTokenPath: {{.DataplaneTokenPath}} +{{end}} {{if .AdminPort }} admin: diff --git a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml index 96682c77a758..77c731c87870 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml @@ -17,6 +17,8 @@ dynamicResources: node: cluster: backend id: default.dp-1.default + metadata: + dataplaneTokenPath: /tmp/token staticResources: clusters: - connectTimeout: 1s diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml new file mode 100644 index 000000000000..94931ca27149 --- /dev/null +++ b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml @@ -0,0 +1,50 @@ +admin: + accessLogPath: /var/log + address: + socketAddress: + address: 192.168.0.1 + portValue: 9902 +dynamicResources: + adsConfig: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: ads_cluster + cdsConfig: + ads: {} + ldsConfig: + ads: {} +node: + cluster: backend + id: mesh.name.namespace +staticResources: + clusters: + - connectTimeout: 2s + http2ProtocolOptions: {} + loadAssignment: + clusterName: ads_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: kuma-control-plane.internal + portValue: 15678 + name: ads_cluster + type: STRICT_DNS + upstreamConnectionOptions: + tcpKeepalive: {} + - connectTimeout: 2s + http2ProtocolOptions: {} + loadAssignment: + clusterName: access_log_sink + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /tmp/kuma-access-logs-name.namespace-mesh.sock + name: access_log_sink + type: STATIC + upstreamConnectionOptions: + tcpKeepalive: {} diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml index 94931ca27149..7b7529488e19 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml @@ -3,7 +3,7 @@ admin: address: socketAddress: address: 192.168.0.1 - portValue: 9902 + portValue: 1234 dynamicResources: adsConfig: apiType: GRPC @@ -17,6 +17,8 @@ dynamicResources: node: cluster: backend id: mesh.name.namespace + metadata: + dataplaneTokenPath: /tmp/token staticResources: clusters: - connectTimeout: 2s diff --git a/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml new file mode 100644 index 000000000000..3f53c55bf0ea --- /dev/null +++ b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml @@ -0,0 +1,44 @@ +dynamicResources: + adsConfig: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: ads_cluster + cdsConfig: + ads: {} + ldsConfig: + ads: {} +node: + cluster: backend + id: mesh.name.namespace +staticResources: + clusters: + - connectTimeout: 1s + http2ProtocolOptions: {} + loadAssignment: + clusterName: ads_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 5678 + name: ads_cluster + type: STRICT_DNS + upstreamConnectionOptions: + tcpKeepalive: {} + - connectTimeout: 1s + http2ProtocolOptions: {} + loadAssignment: + clusterName: access_log_sink + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /tmp/kuma-access-logs-name.namespace-mesh.sock + name: access_log_sink + type: STATIC + upstreamConnectionOptions: + tcpKeepalive: {} diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml index 3f53c55bf0ea..d8b7397eec7e 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml @@ -1,3 +1,9 @@ +admin: + accessLogPath: /dev/null + address: + socketAddress: + address: 127.0.0.1 + portValue: 1234 dynamicResources: adsConfig: apiType: GRPC @@ -11,6 +17,8 @@ dynamicResources: node: cluster: backend id: mesh.name.namespace + metadata: + dataplaneTokenPath: /tmp/token staticResources: clusters: - connectTimeout: 1s diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index b01ef0dbfd43..c754ce7b954a 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -1,7 +1,8 @@ package types type BootstrapRequest struct { - Mesh string `json:"mesh"` - Name string `json:"name"` - AdminPort uint32 `json:"adminPort,omitempty"` + Mesh string `json:"mesh"` + Name string `json:"name"` + AdminPort uint32 `json:"adminPort,omitempty"` + DataplaneTokenPath string `json:"dataplaneTokenPath,omitempty"` } diff --git a/pkg/xds/context/context.go b/pkg/xds/context/context.go index e947b52ae84f..d3e613dce7b9 100644 --- a/pkg/xds/context/context.go +++ b/pkg/xds/context/context.go @@ -5,7 +5,6 @@ import ( "io/ioutil" kuma_cp "github.com/Kong/kuma/pkg/config/app/kuma-cp" - config_core "github.com/Kong/kuma/pkg/config/core" ) type Context struct { @@ -16,8 +15,6 @@ type Context struct { type ControlPlaneContext struct { SdsLocation string SdsTlsCert []byte - - DataplaneTokenFile string } type MeshContext struct { @@ -36,15 +33,9 @@ func BuildControlPlaneContext(config kuma_cp.Config) (*ControlPlaneContext, erro cert = c } sdsLocation := fmt.Sprintf("%s:%d", config.BootstrapServer.Params.XdsHost, config.SdsServer.GrpcPort) - dataplaneTokenFile := "" - if config.Environment == config_core.KubernetesEnvironment { - dataplaneTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" - } return &ControlPlaneContext{ SdsLocation: sdsLocation, SdsTlsCert: cert, - - DataplaneTokenFile: dataplaneTokenFile, }, nil } diff --git a/pkg/xds/envoy/envoy.go b/pkg/xds/envoy/envoy.go index ffb06837138f..3c43516e3da3 100644 --- a/pkg/xds/envoy/envoy.go +++ b/pkg/xds/envoy/envoy.go @@ -93,7 +93,7 @@ func CreateLocalCluster(clusterName string, address string, port uint32) *v2.Clu } } -func CreateEdsCluster(ctx xds_context.Context, clusterName string) *v2.Cluster { +func CreateEdsCluster(ctx xds_context.Context, clusterName string, metadata *core_xds.DataplaneMetadata) *v2.Cluster { connectTimeout := defaultConnectTimeout return &v2.Cluster{ Name: clusterName, @@ -106,7 +106,7 @@ func CreateEdsCluster(ctx xds_context.Context, clusterName string) *v2.Cluster { }, }, }, - TlsContext: CreateUpstreamTlsContext(ctx), + TlsContext: CreateUpstreamTlsContext(ctx, metadata), } } @@ -165,7 +165,7 @@ func CreateOutboundListener(ctx xds_context.Context, listenerName string, addres return listener, nil } -func CreateInboundListener(ctx xds_context.Context, listenerName string, address string, port uint32, clusterName string, virtual bool, permissions *mesh_core.TrafficPermissionResourceList) *v2.Listener { +func CreateInboundListener(ctx xds_context.Context, listenerName string, address string, port uint32, clusterName string, virtual bool, permissions *mesh_core.TrafficPermissionResourceList, metadata *core_xds.DataplaneMetadata) *v2.Listener { config := &tcp.TcpProxy{ StatPrefix: clusterName, ClusterSpecifier: &tcp.TcpProxy_Cluster{ @@ -189,7 +189,7 @@ func CreateInboundListener(ctx xds_context.Context, listenerName string, address }, }, FilterChains: []*envoy_listener.FilterChain{{ - TlsContext: CreateDownstreamTlsContext(ctx), + TlsContext: CreateDownstreamTlsContext(ctx, metadata), Filters: []*envoy_listener.Filter{{ Name: util.TCPProxy, ConfigType: &envoy_listener.Filter_TypedConfig{ @@ -235,47 +235,46 @@ func accessLog(ctx xds_context.Context) []*filter_accesslog.AccessLog { return []*filter_accesslog.AccessLog{logs} } -func CreateDownstreamTlsContext(ctx xds_context.Context) *auth.DownstreamTlsContext { +func CreateDownstreamTlsContext(ctx xds_context.Context, metadata *core_xds.DataplaneMetadata) *auth.DownstreamTlsContext { if !ctx.Mesh.TlsEnabled { return nil } return &auth.DownstreamTlsContext{ - CommonTlsContext: CreateCommonTlsContext(ctx), + CommonTlsContext: CreateCommonTlsContext(ctx, metadata), RequireClientCertificate: &types.BoolValue{Value: true}, } } -func CreateUpstreamTlsContext(ctx xds_context.Context) *auth.UpstreamTlsContext { +func CreateUpstreamTlsContext(ctx xds_context.Context, metadata *core_xds.DataplaneMetadata) *auth.UpstreamTlsContext { if !ctx.Mesh.TlsEnabled { return nil } return &auth.UpstreamTlsContext{ - CommonTlsContext: CreateCommonTlsContext(ctx), + CommonTlsContext: CreateCommonTlsContext(ctx, metadata), } } -func CreateCommonTlsContext(ctx xds_context.Context) *auth.CommonTlsContext { +func CreateCommonTlsContext(ctx xds_context.Context, metadata *core_xds.DataplaneMetadata) *auth.CommonTlsContext { return &auth.CommonTlsContext{ ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{ - ValidationContextSdsSecretConfig: sdsSecretConfig(ctx, server.MeshCaResource), + ValidationContextSdsSecretConfig: sdsSecretConfig(ctx, server.MeshCaResource, metadata), }, TlsCertificateSdsSecretConfigs: []*auth.SdsSecretConfig{ - sdsSecretConfig(ctx, server.IdentityCertResource), + sdsSecretConfig(ctx, server.IdentityCertResource, metadata), }, } } -func sdsSecretConfig(context xds_context.Context, name string) *auth.SdsSecretConfig { +func sdsSecretConfig(context xds_context.Context, name string, metadata *core_xds.DataplaneMetadata) *auth.SdsSecretConfig { withCallCredentials := func(grpc *core.GrpcService_GoogleGrpc) *core.GrpcService_GoogleGrpc { - // TODO(yskopets): credentials should be determined based on properties of a Dataplane rather than global Control Plane settings - if context.ControlPlane.DataplaneTokenFile == "" { + if metadata.DataplaneTokenPath == "" { return grpc } config := &grpc_credential.FileBasedMetadataConfig{ SecretData: &core.DataSource{ Specifier: &core.DataSource_Filename{ - Filename: context.ControlPlane.DataplaneTokenFile, + Filename: metadata.DataplaneTokenPath, }, }, } diff --git a/pkg/xds/envoy/envoy_test.go b/pkg/xds/envoy/envoy_test.go index fffb2ecab12b..08d204519ae9 100644 --- a/pkg/xds/envoy/envoy_test.go +++ b/pkg/xds/envoy/envoy_test.go @@ -89,13 +89,14 @@ var _ = Describe("Envoy", func() { type testCase struct { ctx xds_context.Context + metadata xds.DataplaneMetadata expected string } DescribeTable("should generate 'EDS' Cluster", func(given testCase) { // when - resource := envoy.CreateEdsCluster(given.ctx, "192.168.0.1:8080") + resource := envoy.CreateEdsCluster(given.ctx, "192.168.0.1:8080", &given.metadata) // then actual, err := util_proto.ToYAML(resource) @@ -110,6 +111,7 @@ var _ = Describe("Envoy", func() { TlsEnabled: false, }, }, + metadata: xds.DataplaneMetadata{}, expected: ` connectTimeout: 5s edsClusterConfig: @@ -122,62 +124,14 @@ var _ = Describe("Envoy", func() { Entry("with mTLS", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "", - }, - Mesh: xds_context.MeshContext{ - TlsEnabled: true, - }, - }, - expected: ` - connectTimeout: 5s - edsClusterConfig: - edsConfig: - ads: {} - name: 192.168.0.1:8080 - tlsContext: - commonTlsContext: - tlsCertificateSdsSecretConfigs: - - name: identity_cert - sdsConfig: - apiConfigSource: - apiType: GRPC - grpcServices: - - googleGrpc: - channelCredentials: - sslCredentials: - rootCerts: - inlineBytes: Q0VSVElGSUNBVEU= - statPrefix: sds_identity_cert - targetUri: kuma-control-plane:5677 - validationContextSdsSecretConfig: - name: mesh_ca - sdsConfig: - apiConfigSource: - apiType: GRPC - grpcServices: - - googleGrpc: - channelCredentials: - sslCredentials: - rootCerts: - inlineBytes: Q0VSVElGSUNBVEU= - statPrefix: sds_mesh_ca - targetUri: kuma-control-plane:5677 - type: EDS -`, - }), - Entry("with mTLS", testCase{ - ctx: xds_context.Context{ - ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, }, }, + metadata: xds.DataplaneMetadata{}, expected: ` connectTimeout: 5s edsClusterConfig: @@ -218,14 +172,16 @@ var _ = Describe("Envoy", func() { Entry("with mTLS and Dataplane credentials", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "/var/secret/token", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, }, }, + metadata: xds.DataplaneMetadata{ + DataplaneTokenPath: "/var/secret/token", + }, expected: ` connectTimeout: 5s edsClusterConfig: @@ -319,6 +275,7 @@ var _ = Describe("Envoy", func() { ctx xds_context.Context virtual bool expected string + metadata xds.DataplaneMetadata } DescribeTable("should generate 'inbound' Listener", @@ -359,7 +316,7 @@ var _ = Describe("Envoy", func() { } // when - resource := envoy.CreateInboundListener(given.ctx, "inbound:192.168.0.1:8080", "192.168.0.1", 8080, "localhost:8080", given.virtual, permissions) + resource := envoy.CreateInboundListener(given.ctx, "inbound:192.168.0.1:8080", "192.168.0.1", 8080, "localhost:8080", given.virtual, permissions, &given.metadata) // then actual, err := util_proto.ToYAML(resource) @@ -450,9 +407,8 @@ name: inbound:192.168.0.1:8080 Entry("with mTLS", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, @@ -519,15 +475,17 @@ name: inbound:192.168.0.1:8080 Entry("with mTLS and Dataplane credentials", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "/var/secret/token", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, }, }, virtual: false, + metadata: xds.DataplaneMetadata{ + DataplaneTokenPath: "/var/secret/token", + }, expected: ` address: socketAddress: @@ -700,9 +658,8 @@ name: inbound:192.168.0.1:8080 Entry("with mTLS", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, @@ -727,9 +684,8 @@ name: inbound:192.168.0.1:8080 Entry("with mTLS and Dataplane credentials", testCase{ ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ - SdsLocation: "kuma-control-plane:5677", - SdsTlsCert: []byte("CERTIFICATE"), - DataplaneTokenFile: "/var/secret/token", + SdsLocation: "kuma-control-plane:5677", + SdsTlsCert: []byte("CERTIFICATE"), }, Mesh: xds_context.MeshContext{ TlsEnabled: true, diff --git a/pkg/xds/generator/inbound_proxy_generator_test.go b/pkg/xds/generator/inbound_proxy_generator_test.go index 21645f16e7f8..1bde359b9789 100644 --- a/pkg/xds/generator/inbound_proxy_generator_test.go +++ b/pkg/xds/generator/inbound_proxy_generator_test.go @@ -87,6 +87,7 @@ var _ = Describe("InboundProxyGenerator", func() { }, }, }, + Metadata: &model.DataplaneMetadata{}, } // when diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index 86552b0f4706..f36e5651f6ac 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -73,7 +73,8 @@ var _ = Describe("OutboundProxyGenerator", func() { {Target: "192.168.0.3", Port: 5432}, }, }, - Logs: logs.NewMatchedLogs(), + Logs: logs.NewMatchedLogs(), + Metadata: &model.DataplaneMetadata{}, } // when diff --git a/pkg/xds/generator/proxy_template.go b/pkg/xds/generator/proxy_template.go index 3d13cc7873cc..9859f32fb14e 100644 --- a/pkg/xds/generator/proxy_template.go +++ b/pkg/xds/generator/proxy_template.go @@ -130,7 +130,7 @@ func (_ InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr resources = append(resources, &Resource{ Name: inboundListenerName, Version: "", - Resource: envoy.CreateInboundListener(ctx, inboundListenerName, endpoint.DataplaneIP, endpoint.DataplanePort, localClusterName, virtual, proxy.TrafficPermissions.Get(endpoint.String())), + Resource: envoy.CreateInboundListener(ctx, inboundListenerName, endpoint.DataplaneIP, endpoint.DataplanePort, localClusterName, virtual, proxy.TrafficPermissions.Get(endpoint.String()), proxy.Metadata), }) names[inboundListenerName] = true } @@ -161,7 +161,7 @@ func (_ OutboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.P if used := names[edsClusterName]; !used { resources = append(resources, &Resource{ Name: edsClusterName, - Resource: envoy.CreateEdsCluster(ctx, edsClusterName), + Resource: envoy.CreateEdsCluster(ctx, edsClusterName, proxy.Metadata), }) resources = append(resources, &Resource{ Name: edsClusterName, diff --git a/pkg/xds/generator/proxy_template_profile_source_test.go b/pkg/xds/generator/proxy_template_profile_source_test.go index 01ee9858fa3d..455f5ebe2964 100644 --- a/pkg/xds/generator/proxy_template_profile_source_test.go +++ b/pkg/xds/generator/proxy_template_profile_source_test.go @@ -59,6 +59,7 @@ var _ = Describe("ProxyTemplateProfileSource", func() { Spec: dataplane, }, TrafficPermissions: permissions.MatchedPermissions{}, + Metadata: &model.DataplaneMetadata{}, } // when diff --git a/pkg/xds/generator/proxy_template_test.go b/pkg/xds/generator/proxy_template_test.go index b27401c3e5d0..3d9579af8b14 100644 --- a/pkg/xds/generator/proxy_template_test.go +++ b/pkg/xds/generator/proxy_template_test.go @@ -127,6 +127,7 @@ var _ = Describe("TemplateProxyGenerator", func() { Spec: dataplane, }, TrafficPermissions: permissions.MatchedPermissions{}, + Metadata: &model.DataplaneMetadata{}, } // when diff --git a/pkg/xds/server/components.go b/pkg/xds/server/components.go index 8d95dde5c559..cc14706806d2 100644 --- a/pkg/xds/server/components.go +++ b/pkg/xds/server/components.go @@ -33,12 +33,15 @@ var ( func SetupServer(rt core_runtime.Runtime) error { reconciler := DefaultReconciler(rt) - tracker, err := DefaultDataplaneSyncTracker(rt, reconciler) + metadataTracker := NewDataplaneMetadataTracker() + + tracker, err := DefaultDataplaneSyncTracker(rt, reconciler, metadataTracker) if err != nil { return err } callbacks := util_xds.CallbacksChain{ tracker, + metadataTracker, DefaultDataplaneStatusTracker(rt), } @@ -69,14 +72,14 @@ func DefaultReconciler(rt core_runtime.Runtime) SnapshotReconciler { } } -func DefaultDataplaneSyncTracker(rt core_runtime.Runtime, reconciler SnapshotReconciler) (envoy_xds.Callbacks, error) { +func DefaultDataplaneSyncTracker(rt core_runtime.Runtime, reconciler SnapshotReconciler, metadataTracker *DataplaneMetadataTracker) (envoy_xds.Callbacks, error) { permissionsMatcher := permissions.TrafficPermissionsMatcher{ResourceManager: rt.ResourceManager()} logsMatcher := logs.TrafficLogsMatcher{ResourceManager: rt.ResourceManager()} envoyCpCtx, err := xds_context.BuildControlPlaneContext(rt.Config()) if err != nil { return nil, err } - return xds_sync.NewDataplaneSyncTracker(func(key core_model.ResourceKey) util_watchdog.Watchdog { + return xds_sync.NewDataplaneSyncTracker(func(key core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { log := xdsServerLog.WithName("dataplane-sync-watchdog").WithValues("dataplaneKey", key) return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { @@ -131,6 +134,7 @@ func DefaultDataplaneSyncTracker(rt core_runtime.Runtime, reconciler SnapshotRec TrafficPermissions: matchedPermissions, OutboundTargets: outbound, Logs: matchedLogs, + Metadata: metadataTracker.Metadata(streamId), } return reconciler.Reconcile(envoyCtx, &proxy) }, diff --git a/pkg/xds/server/components_test.go b/pkg/xds/server/components_test.go index 22c1528830ed..1ca5144b7258 100644 --- a/pkg/xds/server/components_test.go +++ b/pkg/xds/server/components_test.go @@ -12,7 +12,6 @@ import ( . "github.com/Kong/kuma/pkg/xds/server" kuma_cp "github.com/Kong/kuma/pkg/config/app/kuma-cp" - core_discovery "github.com/Kong/kuma/pkg/core/discovery" mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" core_model "github.com/Kong/kuma/pkg/core/resources/model" core_store "github.com/Kong/kuma/pkg/core/resources/store" @@ -64,7 +63,7 @@ var _ = Describe("Components", func() { reconciler := eventSnapshotReconciler{} reconciler.events = make(chan event) // and - tracker, err := DefaultDataplaneSyncTracker(runtime, &reconciler) + tracker, err := DefaultDataplaneSyncTracker(runtime, &reconciler, NewDataplaneMetadataTracker()) Expect(err).ToNot(HaveOccurred()) // given @@ -116,23 +115,3 @@ var _ = Describe("Components", func() { }, 10) }) }) - -var _ core_discovery.DataplaneDiscoveryConsumer = DataplaneDiscoveryConsumerFuncs{} - -type DataplaneDiscoveryConsumerFuncs struct { - OnDataplaneUpdateFunc func(*mesh_core.DataplaneResource) error - OnDataplaneDeleteFunc func(core_model.ResourceKey) error -} - -func (f DataplaneDiscoveryConsumerFuncs) OnDataplaneUpdate(dataplane *mesh_core.DataplaneResource) error { - if f.OnDataplaneUpdateFunc != nil { - return f.OnDataplaneUpdateFunc(dataplane) - } - return nil -} -func (f DataplaneDiscoveryConsumerFuncs) OnDataplaneDelete(key core_model.ResourceKey) error { - if f.OnDataplaneDeleteFunc != nil { - return f.OnDataplaneDeleteFunc(key) - } - return nil -} diff --git a/pkg/xds/server/dataplane_metadata_tracker.go b/pkg/xds/server/dataplane_metadata_tracker.go new file mode 100644 index 000000000000..e20fc516bcac --- /dev/null +++ b/pkg/xds/server/dataplane_metadata_tracker.go @@ -0,0 +1,71 @@ +package server + +import ( + "context" + "github.com/Kong/kuma/pkg/core/xds" + "github.com/envoyproxy/go-control-plane/envoy/api/v2" + go_cp_server "github.com/envoyproxy/go-control-plane/pkg/server" + "sync" +) + +type DataplaneMetadataTracker struct { + mutex sync.RWMutex + metadataForStream map[int64]*xds.DataplaneMetadata +} + +func NewDataplaneMetadataTracker() *DataplaneMetadataTracker { + return &DataplaneMetadataTracker{ + mutex: sync.RWMutex{}, + metadataForStream: map[int64]*xds.DataplaneMetadata{}, + } +} + +func (d *DataplaneMetadataTracker) Metadata(streamId int64) *xds.DataplaneMetadata { + d.mutex.RLock() + defer d.mutex.RUnlock() + metadata, found := d.metadataForStream[streamId] + if found { + return metadata + } else { + return &xds.DataplaneMetadata{} + } +} + +var _ go_cp_server.Callbacks = &DataplaneMetadataTracker{} + +func (d *DataplaneMetadataTracker) OnStreamOpen(context.Context, int64, string) error { + return nil +} + +func (d *DataplaneMetadataTracker) OnStreamClosed(stream int64) { + d.mutex.Lock() + defer d.mutex.Unlock() + delete(d.metadataForStream, stream) +} + +func (d *DataplaneMetadataTracker) OnStreamRequest(stream int64, req *v2.DiscoveryRequest) error { + if req.Node == nil { + // from https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-versioning: + // Only the first request on a stream is guaranteed to carry the node identifier. + // The subsequent discovery requests on the same stream may carry an empty node identifier. + // This holds true regardless of the acceptance of the discovery responses on the same stream. + // The node identifier should always be identical if present more than once on the stream. + // It is sufficient to only check the first message for the node identifier as a result. + return nil + } + + d.mutex.Lock() + defer d.mutex.Unlock() + d.metadataForStream[stream] = xds.DataplaneMetadataFromNode(req.Node) + return nil +} + +func (d *DataplaneMetadataTracker) OnStreamResponse(int64, *v2.DiscoveryRequest, *v2.DiscoveryResponse) { +} + +func (d *DataplaneMetadataTracker) OnFetchRequest(context.Context, *v2.DiscoveryRequest) error { + return nil +} + +func (d *DataplaneMetadataTracker) OnFetchResponse(*v2.DiscoveryRequest, *v2.DiscoveryResponse) { +} diff --git a/pkg/xds/server/dataplane_metadata_tracker_test.go b/pkg/xds/server/dataplane_metadata_tracker_test.go new file mode 100644 index 000000000000..7052eb0967e4 --- /dev/null +++ b/pkg/xds/server/dataplane_metadata_tracker_test.go @@ -0,0 +1,73 @@ +package server_test + +import ( + "github.com/Kong/kuma/pkg/core/xds" + "github.com/Kong/kuma/pkg/xds/server" + v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + "github.com/gogo/protobuf/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Dataplane Metadata Tracker", func() { + + tracker := server.NewDataplaneMetadataTracker() + + req := v2.DiscoveryRequest{ + Node: &core.Node{ + Id: "default.example.pilot", + Metadata: &types.Struct{ + Fields: map[string]*types.Value{ + "dataplaneTokenPath": &types.Value{ + Kind: &types.Value_StringValue{ + StringValue: "/tmp/token", + }, + }, + }, + }, + }, + } + const streamId = 123 + + It("should track metadata", func() { + // when + err := tracker.OnStreamRequest(streamId, &req) + + // then + Expect(err).ToNot(HaveOccurred()) + + // when + metadata := tracker.Metadata(streamId) + + // then + Expect(metadata.DataplaneTokenPath).To(Equal("/tmp/token")) + + // when + tracker.OnStreamClosed(streamId) + + // then metadata should be deleted + metadata = tracker.Metadata(streamId) + Expect(metadata).To(Equal(&xds.DataplaneMetadata{})) + }) + + It("should track metadata with empty Node in consecutive DiscoveryRequests", func() { + // when + err := tracker.OnStreamRequest(streamId, &req) + + // then + Expect(err).ToNot(HaveOccurred()) + + // when + err = tracker.OnStreamRequest(streamId, &v2.DiscoveryRequest{}) + + // then + Expect(err).ToNot(HaveOccurred()) + + // when + metadata := tracker.Metadata(streamId) + + // then + Expect(metadata.DataplaneTokenPath).To(Equal("/tmp/token")) + }) +}) diff --git a/pkg/xds/server/snapshot_generator_test.go b/pkg/xds/server/snapshot_generator_test.go index 53e6fa2531db..4fff48d4f15f 100644 --- a/pkg/xds/server/snapshot_generator_test.go +++ b/pkg/xds/server/snapshot_generator_test.go @@ -92,7 +92,8 @@ var _ = Describe("Reconcile", func() { }, }, }, - Logs: logs.NewMatchedLogs(), + Logs: logs.NewMatchedLogs(), + Metadata: &model.DataplaneMetadata{}, } // when diff --git a/pkg/xds/sync/dataplane_sync_tracker.go b/pkg/xds/sync/dataplane_sync_tracker.go index edf098fa1940..bc021aace671 100644 --- a/pkg/xds/sync/dataplane_sync_tracker.go +++ b/pkg/xds/sync/dataplane_sync_tracker.go @@ -17,7 +17,7 @@ var ( dataplaneSyncTrackerLog = core.Log.WithName("xds-server").WithName("dataplane-sync-tracker") ) -type NewDataplaneWatchdogFunc func(dataplaneId core_model.ResourceKey) util_watchdog.Watchdog +type NewDataplaneWatchdogFunc func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog func NewDataplaneSyncTracker(factoryFunc NewDataplaneWatchdogFunc) envoy_xds.Callbacks { return &dataplaneSyncTracker{ @@ -75,7 +75,7 @@ func (t *dataplaneSyncTracker) OnStreamRequest(streamID int64, req *envoy.Discov t.streams[streamID] = context.CancelFunc(func() { close(stopCh) }) - go t.newDataplaneWatchdog(dataplaneKey).Start(stopCh) + go t.newDataplaneWatchdog(dataplaneKey, streamID).Start(stopCh) dataplaneSyncTrackerLog.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", id, "dataplaneKey", dataplaneKey) } else { dataplaneSyncTrackerLog.Error(err, "failed to parse Dataplane Id out of DiscoveryRequest", "streamid", streamID, "req", req) diff --git a/pkg/xds/sync/dataplane_sync_tracker_test.go b/pkg/xds/sync/dataplane_sync_tracker_test.go index bcf574caaac1..c1e38de1471c 100644 --- a/pkg/xds/sync/dataplane_sync_tracker_test.go +++ b/pkg/xds/sync/dataplane_sync_tracker_test.go @@ -70,7 +70,7 @@ var _ = Describe("Sync", func() { watchdogCh := make(chan core_model.ResourceKey) // setup - tracker := NewDataplaneSyncTracker(NewDataplaneWatchdogFunc(func(dataplaneId core_model.ResourceKey) util_watchdog.Watchdog { + tracker := NewDataplaneSyncTracker(NewDataplaneWatchdogFunc(func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { return WatchdogFunc(func(stop <-chan struct{}) { watchdogCh <- dataplaneId <-stop