diff --git a/app/kuma-cp/cmd/run.go b/app/kuma-cp/cmd/run.go index ee8f32e8bfb3..98e9efb2ac8c 100644 --- a/app/kuma-cp/cmd/run.go +++ b/app/kuma-cp/cmd/run.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "time" @@ -46,7 +47,7 @@ func newRunCmd() *cobra.Command { } type runCmdOpts struct { - SetupSignalHandler func() (stopCh <-chan struct{}) + SetupSignalHandler func() context.Context } func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command { @@ -64,8 +65,8 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command { runLog.Error(err, "could not load the configuration") return err } - closeCh := opts.SetupSignalHandler() - rt, err := bootstrap.Bootstrap(cfg, closeCh) + ctx := opts.SetupSignalHandler() + rt, err := bootstrap.Bootstrap(ctx, cfg) if err != nil { runLog.Error(err, "unable to set up Control Plane runtime") return err @@ -184,7 +185,7 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command { } runLog.Info("starting Control Plane", "version", kuma_version.Build.Version) - if err := rt.Start(closeCh); err != nil { + if err := rt.Start(ctx.Done()); err != nil { runLog.Error(err, "problem running Control Plane") return err } diff --git a/app/kuma-cp/cmd/run_test.go b/app/kuma-cp/cmd/run_test.go index bb581a37e590..0b2b8e4e3f71 100644 --- a/app/kuma-cp/cmd/run_test.go +++ b/app/kuma-cp/cmd/run_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "net/http" @@ -32,14 +33,12 @@ func (f ConfigFactoryFunc) GenerateConfig() string { func RunSmokeTest(factory ConfigFactory, workdir string) { Describe("run", func() { - var stopCh chan struct{} var errCh chan error var configFile *os.File var diagnosticsPort int JustBeforeEach(func() { - stopCh = make(chan struct{}) errCh = make(chan error) freePort, _, err := addr.Suggest() @@ -67,9 +66,10 @@ func RunSmokeTest(factory ConfigFactory, workdir string) { config := fmt.Sprintf(factory.GenerateConfig(), diagnosticsPort) _, err := configFile.WriteString(config) Expect(err).ToNot(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) cmd := newRunCmdWithOpts(runCmdOpts{ - SetupSignalHandler: func() <-chan struct{} { - return stopCh + SetupSignalHandler: func() context.Context { + return ctx }, }) cmd.SetArgs([]string{"--config-file=" + configFile.Name()}) @@ -105,7 +105,7 @@ func RunSmokeTest(factory ConfigFactory, workdir string) { // when By("signaling Control Plane to stop") - close(stopCh) + cancel() // then err = <-errCh diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 9fab16c123cc..97d396482fce 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -245,9 +245,9 @@ func writeFile(filename string, data []byte, perm os.FileMode) error { func setupQuitChannel() chan struct{} { quit := make(chan struct{}) - quitOnSignal := core.SetupSignalHandler() + appCtx := core.SetupSignalHandler() go func() { - <-quitOnSignal + <-appCtx.Done() runLog.Info("Kuma DP caught an exit signal") if quit != nil { close(quit) diff --git a/app/kuma-dp/cmd/run_test.go b/app/kuma-dp/cmd/run_test.go index 4dc38a0a777e..50a4d6e30ed6 100644 --- a/app/kuma-dp/cmd/run_test.go +++ b/app/kuma-dp/cmd/run_test.go @@ -3,6 +3,7 @@ package cmd import ( + "context" "fmt" "io" "io/ioutil" @@ -25,26 +26,21 @@ import ( ) var _ = Describe("run", func() { - - var backupSetupSignalHandler func() <-chan struct{} + var backupSetupSignalHandler func() context.Context + var ctx context.Context + var cancel func() BeforeEach(func() { backupSetupSignalHandler = core.SetupSignalHandler + ctx, cancel = context.WithCancel(context.Background()) + core.SetupSignalHandler = func() context.Context { + return ctx + } }) AfterEach(func() { core.SetupSignalHandler = backupSetupSignalHandler }) - var stopCh chan struct{} - - BeforeEach(func() { - stopCh = make(chan struct{}) - - core.SetupSignalHandler = func() <-chan struct{} { - return stopCh - } - }) - var tmpDir string BeforeEach(func() { @@ -156,7 +152,7 @@ var _ = Describe("run", func() { // when By("signaling the dataplane manager to stop") - close(stopCh) + cancel() // then err = <-errCh diff --git a/app/kuma-prometheus-sd/cmd/run.go b/app/kuma-prometheus-sd/cmd/run.go index 5767bb18aac1..ff59610f368b 100644 --- a/app/kuma-prometheus-sd/cmd/run.go +++ b/app/kuma-prometheus-sd/cmd/run.go @@ -68,7 +68,7 @@ func newRunCmd() *cobra.Command { discovery := adapter.NewAdapter(ctx, cfg.Prometheus.OutputFile, "xds_sd", discoverer, util_log.NewLogger(runLog.WithName("xds_sd"), "adapter")) discovery.Run() - <-setupSignalHandler() + <-setupSignalHandler().Done() return nil }, } diff --git a/app/kuma-prometheus-sd/cmd/run_test.go b/app/kuma-prometheus-sd/cmd/run_test.go index 5936296c9b1b..859078ed3c99 100644 --- a/app/kuma-prometheus-sd/cmd/run_test.go +++ b/app/kuma-prometheus-sd/cmd/run_test.go @@ -1,34 +1,30 @@ package cmd import ( + "context" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/pkg/test" ) var _ = Describe("run", func() { - - var backupSetupSignalHandler func() <-chan struct{} + var backupSetupSignalHandler func() context.Context + var ctx context.Context + var cancel func() BeforeEach(func() { - backupSetupSignalHandler = setupSignalHandler + backupSetupSignalHandler = core.SetupSignalHandler + ctx, cancel = context.WithCancel(context.Background()) + core.SetupSignalHandler = func() context.Context { + return ctx + } }) - AfterEach(func() { - setupSignalHandler = backupSetupSignalHandler - }) - - var stopCh chan struct{} - - BeforeEach(func() { - stopCh = make(chan struct{}) - - setupSignalHandler = func() <-chan struct{} { - return stopCh - } + core.SetupSignalHandler = backupSetupSignalHandler }) XIt("should be possible to run `kuma-prometheus-sd run`", test.Within(15*time.Second, func() { @@ -49,7 +45,7 @@ var _ = Describe("run", func() { // when By("signaling Kuma Prometheus SD to stop") - close(stopCh) + cancel() // then err := <-errCh diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml index 1a231bed3829..b6cbb4640c3a 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml @@ -998,7 +998,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -1095,7 +1095,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1115,7 +1115,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1147,7 +1147,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1177,7 +1177,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1214,7 +1214,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1234,7 +1234,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml index 1a64f67f04f5..595615117118 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml @@ -823,7 +823,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -920,7 +920,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -940,7 +940,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -972,7 +972,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1002,7 +1002,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1039,7 +1039,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1059,7 +1059,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml index 52defcfa7a04..a536613d70d1 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml @@ -832,7 +832,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -926,7 +926,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -946,7 +946,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -978,7 +978,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1008,7 +1008,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1045,7 +1045,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1065,7 +1065,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml index 883e9278cf6c..41b8795e3b7a 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml @@ -823,7 +823,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -920,7 +920,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -940,7 +940,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -972,7 +972,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1002,7 +1002,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1039,7 +1039,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1059,7 +1059,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml index 6ae08ae0a20a..ce211504e569 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml @@ -827,7 +827,7 @@ spec: metadata: annotations: checksum/config: 264a6e71a6f440c37bec8b1aa6d668c99c2ae725ecde4051e51f101dda46037b - checksum/tls-secrets: 366bc4ec607782b02e8777186ff838bb56cfc03f8643f832f3403e4c44f00010 + checksum/tls-secrets: dd0b26083d3318a1dfbda240bd513d549121023f9a583bc0ad901627f5f08057 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -964,7 +964,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -984,7 +984,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1016,7 +1016,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Crash clientConfig: caBundle: XYZ @@ -1046,7 +1046,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1083,7 +1083,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1103,7 +1103,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml index bcb0abe8805f..460ff7aac24e 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml @@ -852,7 +852,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -1053,7 +1053,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1073,7 +1073,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1105,7 +1105,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1135,7 +1135,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1172,7 +1172,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1192,7 +1192,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml index c41c2dbf99cc..b0c7c0aa14fc 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml @@ -827,7 +827,7 @@ spec: metadata: annotations: checksum/config: 6df33ec160599ac1a5e9fda109e2ba0d02c865f926ac14ecf92895e96b70a228 - checksum/tls-secrets: 8d96524f2edd766fb63fe05db26d83be9f7d7a5823e0878040d4a6ab95446e78 + checksum/tls-secrets: fbd975366a9b6c3adc0c329ec90b8632d96f24bd6b98ce8e77f4911ef89c8f84 labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -928,7 +928,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -948,7 +948,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -980,7 +980,7 @@ webhooks: sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1010,7 +1010,7 @@ metadata: app.kubernetes.io/instance: kuma webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: XYZ @@ -1047,7 +1047,7 @@ webhooks: sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: XYZ @@ -1067,7 +1067,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/deployments/charts/kuma/templates/cp-webhooks-and-secrets.yaml b/deployments/charts/kuma/templates/cp-webhooks-and-secrets.yaml index af2308075829..c95373343e4b 100644 --- a/deployments/charts/kuma/templates/cp-webhooks-and-secrets.yaml +++ b/deployments/charts/kuma/templates/cp-webhooks-and-secrets.yaml @@ -54,7 +54,7 @@ metadata: {{ include "kuma.labels" . | nindent 4 }} webhooks: - name: mesh.defaulter.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: {{ $caBundle }} @@ -74,7 +74,7 @@ webhooks: - meshes sideEffects: None - name: owner-reference.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: {{ $caBundle }} @@ -105,7 +105,7 @@ webhooks: {{ .Values.controlPlane.webhooks.ownerReference.additionalRules | nindent 6 }} sideEffects: None - name: kuma-injector.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: {{ .Values.controlPlane.injectorFailurePolicy }} clientConfig: caBundle: {{ $caBundle }} @@ -133,7 +133,7 @@ metadata: {{ include "kuma.labels" . | nindent 4 }} webhooks: - name: validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Fail clientConfig: caBundle: {{ $caBundle }} @@ -169,7 +169,7 @@ webhooks: {{ .Values.controlPlane.webhooks.validator.additionalRules | nindent 6 }} sideEffects: None - name: service.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] failurePolicy: Ignore clientConfig: caBundle: {{ $caBundle }} @@ -189,7 +189,7 @@ webhooks: - services sideEffects: None - name: secret.validator.kuma-admission.kuma.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: matchLabels: kuma.io/system-namespace: "true" diff --git a/pkg/core/alias.go b/pkg/core/alias.go index 27d454b71f15..23f02a057c59 100644 --- a/pkg/core/alias.go +++ b/pkg/core/alias.go @@ -11,15 +11,14 @@ import ( ) var ( + // TODO this package should depend on kubernetes stuff Log = kube_log.Log NewLogger = kuma_log.NewLogger NewLoggerWithRotation = kuma_log.NewLoggerWithRotation SetLogger = kube_log.SetLogger Now = time.Now - SetupSignalHandler = func() <-chan struct{} { - return kube_signals.SetupSignalHandler().Done() - } + SetupSignalHandler = kube_signals.SetupSignalHandler NewUUID = func() string { return string(kube_uuid.NewUUID()) diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go index ff66fb7a74b8..51cf00fddd19 100644 --- a/pkg/core/bootstrap/bootstrap.go +++ b/pkg/core/bootstrap/bootstrap.go @@ -44,11 +44,11 @@ import ( "github.com/kumahq/kuma/pkg/xds/secrets" ) -func buildRuntime(cfg kuma_cp.Config, closeCh <-chan struct{}) (core_runtime.Runtime, error) { +func buildRuntime(appCtx context.Context, cfg kuma_cp.Config) (core_runtime.Runtime, error) { if err := autoconfigure(&cfg); err != nil { return nil, err } - builder, err := core_runtime.BuilderFor(cfg, closeCh) + builder, err := core_runtime.BuilderFor(appCtx, cfg) if err != nil { return nil, err } @@ -140,8 +140,8 @@ func initializeMetrics(builder *core_runtime.Builder) error { return nil } -func Bootstrap(cfg kuma_cp.Config, closeCh <-chan struct{}) (core_runtime.Runtime, error) { - runtime, err := buildRuntime(cfg, closeCh) +func Bootstrap(ctx context.Context, cfg kuma_cp.Config) (core_runtime.Runtime, error) { + runtime, err := buildRuntime(ctx, cfg) if err != nil { return nil, err } diff --git a/pkg/core/runtime/builder.go b/pkg/core/runtime/builder.go index 0845b8d98076..acc576cafa49 100644 --- a/pkg/core/runtime/builder.go +++ b/pkg/core/runtime/builder.go @@ -56,34 +56,34 @@ var _ BuilderContext = &Builder{} // Builder represents a multi-step initialization process. type Builder struct { - cfg kuma_cp.Config - cm component.Manager - rs core_store.ResourceStore - ss store.SecretStore - cs core_store.ResourceStore - rm core_manager.CustomizableResourceManager - rom core_manager.ReadOnlyResourceManager - cam core_ca.Managers - dsl datasource.Loader - ext context.Context - dns resolver.DNSResolver - configm config_manager.ConfigManager - leadInfo component.LeaderInfo - lif lookup.LookupIPFunc - eac admin.EnvoyAdminClient - metrics metrics.Metrics - erf events.ListenerFactory - apim api_server.APIManager - xdsh *xds_hooks.Hooks - cap secrets.CaProvider - dps *dp_server.DpServer - kdsctx *kds_context.Context - mv core_managers.MeshValidator - shutdownCh <-chan struct{} + cfg kuma_cp.Config + cm component.Manager + rs core_store.ResourceStore + ss store.SecretStore + cs core_store.ResourceStore + rm core_manager.CustomizableResourceManager + rom core_manager.ReadOnlyResourceManager + cam core_ca.Managers + dsl datasource.Loader + ext context.Context + dns resolver.DNSResolver + configm config_manager.ConfigManager + leadInfo component.LeaderInfo + lif lookup.LookupIPFunc + eac admin.EnvoyAdminClient + metrics metrics.Metrics + erf events.ListenerFactory + apim api_server.APIManager + xdsh *xds_hooks.Hooks + cap secrets.CaProvider + dps *dp_server.DpServer + kdsctx *kds_context.Context + mv core_managers.MeshValidator + appCtx context.Context *runtimeInfo } -func BuilderFor(cfg kuma_cp.Config, closeCh <-chan struct{}) (*Builder, error) { +func BuilderFor(appCtx context.Context, cfg kuma_cp.Config) (*Builder, error) { hostname, err := os.Hostname() if err != nil { return nil, errors.Wrap(err, "could not get hostname") @@ -96,7 +96,7 @@ func BuilderFor(cfg kuma_cp.Config, closeCh <-chan struct{}) (*Builder, error) { runtimeInfo: &runtimeInfo{ instanceId: fmt.Sprintf("%s-%s", hostname, suffix), }, - shutdownCh: closeCh, + appCtx: appCtx, }, nil } @@ -278,28 +278,28 @@ func (b *Builder) Build() (Runtime, error) { return &runtime{ RuntimeInfo: b.runtimeInfo, RuntimeContext: &runtimeContext{ - cfg: b.cfg, - rm: b.rm, - rom: b.rom, - rs: b.rs, - ss: b.ss, - cam: b.cam, - dsl: b.dsl, - ext: b.ext, - dns: b.dns, - configm: b.configm, - leadInfo: b.leadInfo, - lif: b.lif, - eac: b.eac, - metrics: b.metrics, - erf: b.erf, - apim: b.apim, - xdsh: b.xdsh, - cap: b.cap, - dps: b.dps, - kdsctx: b.kdsctx, - mv: b.mv, - shutdownCh: b.shutdownCh, + cfg: b.cfg, + rm: b.rm, + rom: b.rom, + rs: b.rs, + ss: b.ss, + cam: b.cam, + dsl: b.dsl, + ext: b.ext, + dns: b.dns, + configm: b.configm, + leadInfo: b.leadInfo, + lif: b.lif, + eac: b.eac, + metrics: b.metrics, + erf: b.erf, + apim: b.apim, + xdsh: b.xdsh, + cap: b.cap, + dps: b.dps, + kdsctx: b.kdsctx, + mv: b.mv, + appCtx: b.appCtx, }, Manager: b.cm, }, nil @@ -372,5 +372,8 @@ func (b *Builder) MeshValidator() core_managers.MeshValidator { return b.mv } func (b *Builder) ShutdownCh() <-chan struct{} { - return b.shutdownCh + return b.appCtx.Done() +} +func (b *Builder) AppContext() context.Context { + return b.appCtx } diff --git a/pkg/core/runtime/runtime.go b/pkg/core/runtime/runtime.go index 2589fba7e3a9..6eca5ec757b6 100644 --- a/pkg/core/runtime/runtime.go +++ b/pkg/core/runtime/runtime.go @@ -61,7 +61,7 @@ type RuntimeContext interface { DpServer() *dp_server.DpServer KDSContext() *kds_context.Context MeshValidator() core_managers.MeshValidator - ShutdownCh() <-chan struct{} + AppContext() context.Context } var _ Runtime = &runtime{} @@ -100,29 +100,29 @@ func (i *runtimeInfo) GetClusterId() string { var _ RuntimeContext = &runtimeContext{} type runtimeContext struct { - cfg kuma_cp.Config - rm core_manager.ResourceManager - rs core_store.ResourceStore - ss store.SecretStore - cs core_store.ResourceStore - rom core_manager.ReadOnlyResourceManager - cam ca.Managers - dsl datasource.Loader - ext context.Context - dns resolver.DNSResolver - configm config_manager.ConfigManager - leadInfo component.LeaderInfo - lif lookup.LookupIPFunc - eac admin.EnvoyAdminClient - metrics metrics.Metrics - erf events.ListenerFactory - apim api_server.APIInstaller - xdsh *xds_hooks.Hooks - cap secrets.CaProvider - dps *dp_server.DpServer - kdsctx *kds_context.Context - mv core_managers.MeshValidator - shutdownCh <-chan struct{} + cfg kuma_cp.Config + rm core_manager.ResourceManager + rs core_store.ResourceStore + ss store.SecretStore + cs core_store.ResourceStore + rom core_manager.ReadOnlyResourceManager + cam ca.Managers + dsl datasource.Loader + ext context.Context + dns resolver.DNSResolver + configm config_manager.ConfigManager + leadInfo component.LeaderInfo + lif lookup.LookupIPFunc + eac admin.EnvoyAdminClient + metrics metrics.Metrics + erf events.ListenerFactory + apim api_server.APIInstaller + xdsh *xds_hooks.Hooks + cap secrets.CaProvider + dps *dp_server.DpServer + kdsctx *kds_context.Context + mv core_managers.MeshValidator + appCtx context.Context } func (rc *runtimeContext) Metrics() metrics.Metrics { @@ -212,6 +212,6 @@ func (rc *runtimeContext) MeshValidator() core_managers.MeshValidator { return rc.mv } -func (rc *runtimeContext) ShutdownCh() <-chan struct{} { - return rc.shutdownCh +func (b *runtimeContext) AppContext() context.Context { + return b.appCtx } diff --git a/pkg/plugins/bootstrap/k8s/cache/internal/deleg_map.go b/pkg/plugins/bootstrap/k8s/cache/internal/deleg_map.go index a900f18dd4e7..d815bb12745d 100644 --- a/pkg/plugins/bootstrap/k8s/cache/internal/deleg_map.go +++ b/pkg/plugins/bootstrap/k8s/cache/internal/deleg_map.go @@ -49,8 +49,8 @@ func NewInformersMap(config *rest.Config, namespace string, ) *InformersMap { return &InformersMap{ - structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace), - unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace), + structured: newSpecificInformersMap(config, scheme, mapper, resync, namespace, createStructuredListWatch), + unstructured: newSpecificInformersMap(config, scheme, mapper, resync, namespace, createUnstructuredListWatch), Scheme: scheme, } @@ -91,13 +91,3 @@ func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj return m.structured.Get(ctx, gvk, obj) } - -// newStructuredInformersMap creates a new InformersMap for structured objects. -func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createStructuredListWatch) -} - -// newUnstructuredInformersMap creates a new InformersMap for unstructured objects. -func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createUnstructuredListWatch) -} diff --git a/pkg/plugins/bootstrap/k8s/plugin.go b/pkg/plugins/bootstrap/k8s/plugin.go index 289a20823b78..3dc0a04bbfcd 100644 --- a/pkg/plugins/bootstrap/k8s/plugin.go +++ b/pkg/plugins/bootstrap/k8s/plugin.go @@ -9,11 +9,13 @@ import ( "github.com/pkg/errors" kube_core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kube_runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" kube_ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" kube_client "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" kube_manager "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/kumahq/kuma/pkg/core" @@ -58,7 +60,7 @@ func (p *plugin) BeforeBootstrap(b *core_runtime.Builder, _ core_plugins.PluginC return err } - secretClient, err := secretClient(b.Config().Store.Kubernetes.SystemNamespace, config, scheme, mgr.GetRESTMapper(), b.ShutdownCh()) + secretClient, err := createSecretClient(b.AppContext(), b.Config().Store.Kubernetes.SystemNamespace, config, mgr.GetRESTMapper()) if err != nil { return err } @@ -80,29 +82,23 @@ func (p *plugin) BeforeBootstrap(b *core_runtime.Builder, _ core_plugins.PluginC // If we try to use regular cached client for Secrets then we will see following error: E1126 10:42:52.097662 1 reflector.go:178] pkg/mod/k8s.io/client-go@v0.18.9/tools/cache/reflector.go:125: Failed to list *v1.Secret: secrets is forbidden: User "system:serviceaccount:kuma-system:kuma-control-plane" cannot list resource "secrets" in API group "" at the cluster scope // We cannot specify this Namespace parameter for the main cache in ControllerManager because it affect all the resources, therefore we need separate client with cache for Secrets. // The alternative was to use non-cached client, but it had performance problems. -func secretClient(systemNamespace string, config *rest.Config, scheme *kube_runtime.Scheme, restMapper meta.RESTMapper, closeCh <-chan struct{}) (kube_client.Client, error) { - ctx, cancel := context.WithCancel(context.Background()) - go func() { - <-closeCh - cancel() - }() +func createSecretClient(ctx context.Context, systemNamespace string, config *rest.Config, restMapper meta.RESTMapper) (kube_client.Client, error) { resyncPeriod := 10 * time.Hour // default resyncPeriod in Kubernetes - kubeCache, err := kuma_kube_cache.New(config, cache.Options{ - Scheme: scheme, - Mapper: restMapper, - Resync: &resyncPeriod, - Namespace: systemNamespace, + scheme := kube_runtime.NewScheme() + kubeCache, err := cache.MultiNamespacedCacheBuilder([]string{systemNamespace})(config, cache.Options{ + Scheme: scheme, + Mapper: restMapper, + Resync: &resyncPeriod, }) if err != nil { return nil, err } // Add kube core scheme first, otherwise cache won't start - if err := kube_core.AddToScheme(scheme); err != nil { - return nil, errors.Wrapf(err, "could not add %q to scheme", kube_core.SchemeGroupVersion) - } + scheme.AddKnownTypes(kube_core.SchemeGroupVersion, &kube_core.Secret{}, &kube_core.SecretList{}, &metav1.Status{}) + metav1.AddToGroupVersion(scheme, kube_core.SchemeGroupVersion) // We are listing secrets by our custom "type", therefore we need to add index by this field into cache - err = kubeCache.IndexField(context.Background(), &kube_core.Secret{}, "type", func(object kube_client.Object) []string { + err = kubeCache.IndexField(ctx, &kube_core.Secret{}, "type", func(object kube_client.Object) []string { secret := object.(*kube_core.Secret) return []string{string(secret.Type)} }) @@ -125,7 +121,10 @@ func secretClient(systemNamespace string, config *rest.Config, scheme *kube_runt core.Log.Error(errors.New("could not sync secret cache"), "failed to wait for cache") } - return kube_client.New(config, kube_client.Options{Scheme: scheme, Mapper: restMapper}) + return cluster.DefaultNewClient(kubeCache, config, kube_client.Options{ + Scheme: scheme, + Mapper: restMapper, + }) } func (p *plugin) AfterBootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error { diff --git a/pkg/plugins/runtime/k8s/webhooks/secret_validator.go b/pkg/plugins/runtime/k8s/webhooks/secret_validator.go index be7ad809edcf..43760b334bde 100644 --- a/pkg/plugins/runtime/k8s/webhooks/secret_validator.go +++ b/pkg/plugins/runtime/k8s/webhooks/secret_validator.go @@ -24,7 +24,7 @@ const ( type SecretValidator struct { Decoder *admission.Decoder - Client kube_client.Client + Client kube_client.Reader Validator secret_manager.SecretValidator } diff --git a/pkg/test/runtime/runtime.go b/pkg/test/runtime/runtime.go index b53cb66ea993..783b2c2530e1 100644 --- a/pkg/test/runtime/runtime.go +++ b/pkg/test/runtime/runtime.go @@ -1,6 +1,7 @@ package runtime import ( + "context" "net" "github.com/kumahq/kuma/pkg/api-server/customization" @@ -50,8 +51,7 @@ func (i *TestRuntimeInfo) GetClusterId() string { } func BuilderFor(cfg kuma_cp.Config) (*core_runtime.Builder, error) { - stopCh := make(chan struct{}) - builder, err := core_runtime.BuilderFor(cfg, stopCh) + builder, err := core_runtime.BuilderFor(context.Background(), cfg) if err != nil { return nil, err } diff --git a/pkg/xds/server/v3/components.go b/pkg/xds/server/v3/components.go index f589f5e47a6d..91e03ecc4169 100644 --- a/pkg/xds/server/v3/components.go +++ b/pkg/xds/server/v3/components.go @@ -57,7 +57,7 @@ func RegisterXDS( util_xds_v3.AdaptCallbacks(authCallbacks), util_xds_v3.AdaptCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(xds_callbacks.NewDataplaneSyncTracker(watchdogFactory.New))), util_xds_v3.AdaptCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(metadataTracker)), - util_xds_v3.AdaptCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(xds_callbacks.NewDataplaneLifecycle(rt.ResourceManager(), rt.ShutdownCh()))), + util_xds_v3.AdaptCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(xds_callbacks.NewDataplaneLifecycle(rt.ResourceManager(), rt.AppContext().Done()))), util_xds_v3.AdaptCallbacks(DefaultDataplaneStatusTracker(rt, envoyCpCtx.Secrets)), util_xds_v3.AdaptCallbacks(xds_callbacks.NewNackBackoff(rt.Config().XdsServer.NACKBackoff)), newResourceWarmingForcer(xdsContext.Cache(), xdsContext.Hasher()),