diff --git a/Makefile b/Makefile index 8c536e521..ddaf97c45 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,9 @@ start-provider-gcp: -ldflags $(LD_FLAGS) \ ./controllers/provider-gcp/cmd/gardener-extension-provider-gcp \ --leader-election=$(LEADER_ELECTION) \ + --webhook-config-mode=url \ + --webhook-config-name=gcp-webhooks \ + --webhook-config-host=$(HOSTNAME) \ --infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION) .PHONY: start-provider-openstack diff --git a/controllers/provider-aws/pkg/webhook/controlplane/add.go b/controllers/provider-aws/pkg/webhook/controlplane/add.go index 0798604f2..6c39efa53 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/add.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/add.go @@ -36,6 +36,6 @@ func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { Kind: extensionswebhook.ShootKind, Provider: aws.Type, Types: []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}}, - Mutator: newMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), + Mutator: NewMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), }) } diff --git a/controllers/provider-aws/pkg/webhook/controlplane/mutator.go b/controllers/provider-aws/pkg/webhook/controlplane/mutator.go index d124aa604..0305957b8 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/mutator.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/mutator.go @@ -31,8 +31,8 @@ import ( kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" ) -// newMutator creates a new controlplane mutator. -func newMutator(unitSerializer controlplane.UnitSerializer, kubeletConfigCodec controlplane.KubeletConfigCodec, logger logr.Logger) *mutator { +// NewMutator creates a new controlplane mutator. +func NewMutator(unitSerializer controlplane.UnitSerializer, kubeletConfigCodec controlplane.KubeletConfigCodec, logger logr.Logger) controlplane.Mutator { return &mutator{ unitSerializer: unitSerializer, kubeletConfigCodec: kubeletConfigCodec, diff --git a/controllers/provider-aws/pkg/webhook/controlplane/mutator_test.go b/controllers/provider-aws/pkg/webhook/controlplane/mutator_test.go index 5b61e0cc2..a8c5ca432 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/mutator_test.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/mutator_test.go @@ -19,15 +19,29 @@ import ( "testing" "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws" + mockcontrolplane "github.com/gardener/gardener-extensions/pkg/mock/gardener-extensions/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/util" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test" + "github.com/coreos/go-systemd/unit" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +const ( + oldServiceContent = "old kubelet.service content" + newServiceContent = "new kubelet.service content" + + oldKubeletConfigData = "old kubelet config data" + newKubeletConfigData = "new kubelet config data" ) func TestController(t *testing.T) { @@ -36,6 +50,16 @@ func TestController(t *testing.T) { } var _ = Describe("Mutator", func() { + var ( + ctrl *gomock.Controller + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + AfterEach(func() { + ctrl.Finish() + }) Describe("#Mutate", func() { It("should add missing elements to kube-apiserver deployment", func() { @@ -56,7 +80,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(nil, nil, logger) + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeAPIServerDeployment(dep) @@ -100,7 +127,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(nil, nil, logger) + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeAPIServerDeployment(dep) @@ -124,7 +154,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(nil, nil, logger) + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeControllerManagerDeployment(dep) @@ -165,11 +198,90 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(nil, nil, logger) + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeControllerManagerDeployment(dep) }) + + It("should modify existing elements of OperatingSystemConfig", func() { + var ( + osc = &extensionsv1alpha1.OperatingSystemConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: extensionsv1alpha1.OperatingSystemConfigSpec{ + Purpose: extensionsv1alpha1.OperatingSystemConfigPurposeReconcile, + Units: []extensionsv1alpha1.Unit{ + { + Name: "kubelet.service", + Content: util.StringPtr(oldServiceContent), + }, + }, + Files: []extensionsv1alpha1.File{ + { + Path: "/var/lib/kubelet/config/kubelet", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Data: oldKubeletConfigData, + }, + }, + }, + }, + }, + } + + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --cloud-provider=aws`, + }, + } + + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + ) + + // Create mock UnitSerializer + us := mockcontrolplane.NewMockUnitSerializer(ctrl) + us.EXPECT().Deserialize(oldServiceContent).Return(oldUnitOptions, nil) + us.EXPECT().Serialize(newUnitOptions).Return(newServiceContent, nil) + + // Create mock KubeletConfigCodec + kcc := mockcontrolplane.NewMockKubeletConfigCodec(ctrl) + kcc.EXPECT().Decode(&extensionsv1alpha1.FileContentInline{Data: oldKubeletConfigData}).Return(oldKubeletConfig, nil) + kcc.EXPECT().Encode(newKubeletConfig, "").Return(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData}, nil) + + // Create mutator + mutator := NewMutator(us, kcc, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), osc) + Expect(err).To(Not(HaveOccurred())) + checkOperatingSystemConfig(osc) + }) }) }) @@ -209,3 +321,12 @@ func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) } + +func checkOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) { + u := controlplane.UnitWithName(osc.Spec.Units, "kubelet.service") + Expect(u).To(Not(BeNil())) + Expect(u.Content).To(Equal(util.StringPtr(newServiceContent))) + f := controlplane.FileWithPath(osc.Spec.Files, "/var/lib/kubelet/config/kubelet") + Expect(f).To(Not(BeNil())) + Expect(f.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData})) +} diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go index 5838d4efc..1f5edf73e 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go @@ -36,6 +36,6 @@ func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { Kind: extensionswebhook.SeedKind, Provider: aws.Type, Types: []runtime.Object{&corev1.Service{}, &appsv1.Deployment{}}, - Mutator: newMutator(logger), + Mutator: NewMutator(logger), }) } diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator.go index 907af8f78..dcccfb727 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator.go @@ -26,8 +26,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -// newMutator creates a new controlplaneexposure mutator. -func newMutator(logger logr.Logger) *mutator { +// NewMutator creates a new controlplaneexposure mutator. +func NewMutator(logger logr.Logger) controlplane.Mutator { return &mutator{ logger: logger.WithName("mutator"), } @@ -38,7 +38,7 @@ type mutator struct { } // Mutate validates and if needed mutates the given object. -func (v *mutator) Mutate(ctx context.Context, obj runtime.Object) error { +func (m *mutator) Mutate(ctx context.Context, obj runtime.Object) error { switch x := obj.(type) { case *corev1.Service: switch x.Name { diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator_test.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator_test.go index ef514faf6..6b88bfb62 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator_test.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator_test.go @@ -43,7 +43,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(logger) + // Create mutator + mutator := NewMutator(logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), svc) Expect(err).To(Not(HaveOccurred())) Expect(svc.Annotations).To(HaveKeyWithValue("service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout", "3600")) @@ -63,7 +66,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(logger) + // Create mutator + mutator := NewMutator(logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), svc) Expect(err).To(Not(HaveOccurred())) Expect(svc.Annotations).To(BeEmpty()) @@ -87,7 +93,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(logger) + // Create mutator + mutator := NewMutator(logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeAPIServerDeployment(dep) @@ -112,7 +121,10 @@ var _ = Describe("Mutator", func() { } ) - mutator := newMutator(logger) + // Create mutator + mutator := NewMutator(logger) + + // Call Mutate method and check the result err := mutator.Mutate(context.TODO(), dep) Expect(err).To(Not(HaveOccurred())) checkKubeAPIServerDeployment(dep) diff --git a/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go b/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go index 262fc6e5f..05ee5cfd6 100644 --- a/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go +++ b/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go @@ -23,9 +23,11 @@ import ( gcpcontroller "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller" gcpcontrolplane "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/controlplane" gcpinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/infrastructure" + gcpwebhook "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -58,7 +60,17 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } controlPlaneOpts = controllercmd.PrefixOption("controlplane-", controlPlaneCtrlOpts) - aggOption = controllercmd.NewOptionAggregator(restOpts, mgrOpts, infraOpts, controlPlaneOpts) + webhookServerOpts = &webhookcmd.WebhookServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + + aggOption = controllercmd.NewOptionAggregator(restOpts, mgrOpts, infraOpts, controlPlaneOpts, webhookServerOpts) ) cmd := &cobra.Command{ @@ -90,6 +102,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } + if err := gcpwebhook.AddToManager(mgr, webhookServerOpts.Completed()); err != nil { + controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") + } + if err := mgr.Start(ctx.Done()); err != nil { controllercmd.LogErrAndExit(err, "Error running manager") } diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/add.go b/controllers/provider-gcp/pkg/webhook/controlplane/add.go new file mode 100644 index 000000000..285212c50 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplane/add.go @@ -0,0 +1,41 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = log.Log.WithName("gcp-controlplane-webhook") + +// AddToManager adds a webhook to the given manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.ShootKind, + Provider: gcp.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}}, + Mutator: NewMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), + }) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/mutator.go b/controllers/provider-gcp/pkg/webhook/controlplane/mutator.go new file mode 100644 index 000000000..04ef353cd --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplane/mutator.go @@ -0,0 +1,242 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "fmt" + + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/coreos/go-systemd/unit" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// NewMutator creates a new controlplane mutator. +func NewMutator(unitSerializer controlplane.UnitSerializer, kubeletConfigCodec controlplane.KubeletConfigCodec, logger logr.Logger) controlplane.Mutator { + return &mutator{ + unitSerializer: unitSerializer, + kubeletConfigCodec: kubeletConfigCodec, + logger: logger.WithName("mutator"), + } +} + +type mutator struct { + unitSerializer controlplane.UnitSerializer + kubeletConfigCodec controlplane.KubeletConfigCodec + logger logr.Logger +} + +// Mutate validates and if needed mutates the given object. +func (m *mutator) Mutate(ctx context.Context, obj runtime.Object) error { + switch x := obj.(type) { + case *appsv1.Deployment: + switch x.Name { + case common.KubeAPIServerDeploymentName: + return mutateKubeAPIServerDeployment(x) + case common.KubeControllerManagerDeploymentName: + return mutateKubeControllerManagerDeployment(x) + } + case *extensionsv1alpha1.OperatingSystemConfig: + if x.Spec.Purpose == extensionsv1alpha1.OperatingSystemConfigPurposeReconcile { + return m.mutateOperatingSystemConfig(x) + } + } + return nil +} + +func mutateKubeAPIServerDeployment(dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-apiserver"); c != nil { + ensureKubeAPIServerCommandLineArgs(c) + ensureEnvVars(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +func mutateKubeControllerManagerDeployment(dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-controller-manager"); c != nil { + ensureKubeControllerManagerCommandLineArgs(c) + ensureEnvVars(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "gce") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = controlplane.EnsureNoStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") +} + +func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-cloud-volume-plugin=", "gce") +} + +var ( + credentialsEnvVar = corev1.EnvVar{ + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: fmt.Sprintf("/srv/cloudprovider/%s", gcp.ServiceAccountJSONField), + } +) + +func ensureEnvVars(c *corev1.Container) { + c.Env = controlplane.EnsureEnvVarWithName(c.Env, credentialsEnvVar) +} + +var ( + cloudProviderConfigVolumeMount = corev1.VolumeMount{ + Name: internal.CloudProviderConfigName, + MountPath: "/etc/kubernetes/cloudprovider", + } + cloudProviderSecretVolumeMount = corev1.VolumeMount{ + Name: common.CloudProviderSecretName, + MountPath: "/srv/cloudprovider", + } + + cloudProviderConfigVolume = corev1.Volume{ + Name: internal.CloudProviderConfigName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: internal.CloudProviderConfigName}, + }, + }, + } + cloudProviderSecretVolume = corev1.Volume{ + Name: common.CloudProviderSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + SecretName: common.CloudProviderSecretName, + }, + }, + } +) + +func ensureVolumeMounts(c *corev1.Container) { + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderConfigVolumeMount) + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderSecretVolumeMount) +} + +func ensureVolumes(ps *corev1.PodSpec) { + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderConfigVolume) + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderSecretVolume) +} + +func (m *mutator) mutateOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) error { + // Mutate kubelet.service unit, if present + if u := controlplane.UnitWithName(osc.Spec.Units, "kubelet.service"); u != nil && u.Content != nil { + if err := m.ensureKubeletServiceUnitContent(u.Content); err != nil { + return err + } + } + + // Mutate kubelet configuration file, if present + if f := controlplane.FileWithPath(osc.Spec.Files, "/var/lib/kubelet/config/kubelet"); f != nil && f.Content.Inline != nil { + if err := m.ensureKubeletConfigFileContent(f.Content.Inline); err != nil { + return err + } + } + + return nil +} + +func (m *mutator) ensureKubeletServiceUnitContent(content *string) error { + var opts []*unit.UnitOption + var err error + + // Deserialize unit options + if opts, err = m.unitSerializer.Deserialize(*content); err != nil { + return errors.Wrap(err, "could not deserialize kubelet.service unit content") + } + + opts = ensureKubeletServiceUnitOptions(opts) + + // Serialize unit options + if *content, err = m.unitSerializer.Serialize(opts); err != nil { + return errors.Wrap(err, "could not serialize kubelet.service unit options") + } + + return nil +} + +func (m *mutator) ensureKubeletConfigFileContent(fci *extensionsv1alpha1.FileContentInline) error { + var kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration + var err error + + // Decode kubelet configuration from inline content + if kubeletConfig, err = m.kubeletConfigCodec.Decode(fci); err != nil { + return errors.Wrap(err, "could not decode kubelet configuration") + } + + ensureKubeletConfiguration(kubeletConfig) + + // Encode kubelet configuration into inline content + var newFCI *extensionsv1alpha1.FileContentInline + if newFCI, err = m.kubeletConfigCodec.Encode(kubeletConfig, fci.Encoding); err != nil { + return errors.Wrap(err, "could not encode kubelet configuration") + } + *fci = *newFCI + + return nil +} + +func ensureKubeletServiceUnitOptions(opts []*unit.UnitOption) []*unit.UnitOption { + if opt := controlplane.UnitOptionWithSectionAndName(opts, "Service", "ExecStart"); opt != nil { + command := controlplane.DeserializeCommandLine(opt.Value) + command = ensureKubeletCommandLineArgs(command) + opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n ") + } + opts = controlplane.EnsureUnitOption(opts, &unit.UnitOption{ + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(echo $HOSTNAME | cut -d '.' -f 1)'`, + }) + return opts +} + +func ensureKubeletCommandLineArgs(command []string) []string { + command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "gce") + return command +} + +func ensureKubeletConfiguration(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) { + // Make sure CSI-related feature gates are not enabled + // TODO Leaving these enabled shouldn't do any harm, perhaps remove this code when properly tested? + delete(kubeletConfig.FeatureGates, "VolumeSnapshotDataSource") + delete(kubeletConfig.FeatureGates, "CSINodeInfo") + delete(kubeletConfig.FeatureGates, "CSIDriverRegistry") +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/mutator_test.go b/controllers/provider-gcp/pkg/webhook/controlplane/mutator_test.go new file mode 100644 index 000000000..07e3a0b21 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplane/mutator_test.go @@ -0,0 +1,333 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal" + mockcontrolplane "github.com/gardener/gardener-extensions/pkg/mock/gardener-extensions/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test" + + "github.com/coreos/go-systemd/unit" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +const ( + oldServiceContent = "old kubelet.service content" + newServiceContent = "new kubelet.service content" + + oldKubeletConfigData = "old kubelet config data" + newKubeletConfigData = "new kubelet config data" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "GCP Controlplane Webhook Suite") +} + +var _ = Describe("Mutator", func() { + var ( + ctrl *gomock.Controller + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#Mutate", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--enable-admission-plugins=Priority,NamespaceLifecycle", + "--disable-admission-plugins=PersistentVolumeLabel", + }, + Env: []corev1.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "?"}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: internal.CloudProviderConfigName, MountPath: "?"}, + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: internal.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should add missing elements to kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + }, + }, + }, + }, + }, + } + ) + + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--external-cloud-volume-plugin=?", + }, + Env: []corev1.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "?"}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: internal.CloudProviderConfigName, MountPath: "?"}, + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: internal.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create mutator + mutator := NewMutator(nil, nil, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of OperatingSystemConfig", func() { + var ( + osc = &extensionsv1alpha1.OperatingSystemConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: extensionsv1alpha1.OperatingSystemConfigSpec{ + Purpose: extensionsv1alpha1.OperatingSystemConfigPurposeReconcile, + Units: []extensionsv1alpha1.Unit{ + { + Name: "kubelet.service", + Content: util.StringPtr(oldServiceContent), + }, + }, + Files: []extensionsv1alpha1.File{ + { + Path: "/var/lib/kubelet/config/kubelet", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Data: oldKubeletConfigData, + }, + }, + }, + }, + }, + } + + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --cloud-provider=gce`, + }, + { + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(echo $HOSTNAME | cut -d '.' -f 1)'`, + }, + } + + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + ) + + // Create mock UnitSerializer + us := mockcontrolplane.NewMockUnitSerializer(ctrl) + us.EXPECT().Deserialize(oldServiceContent).Return(oldUnitOptions, nil) + us.EXPECT().Serialize(newUnitOptions).Return(newServiceContent, nil) + + // Create mock KubeletConfigCodec + kcc := mockcontrolplane.NewMockKubeletConfigCodec(ctrl) + kcc.EXPECT().Decode(&extensionsv1alpha1.FileContentInline{Data: oldKubeletConfigData}).Return(oldKubeletConfig, nil) + kcc.EXPECT().Encode(newKubeletConfig, "").Return(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData}, nil) + + // Create mutator + mutator := NewMutator(us, kcc, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), osc) + Expect(err).To(Not(HaveOccurred())) + checkOperatingSystemConfig(osc) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=gce")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ",")) + Expect(c.Command).To(Not(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ","))) + Expect(c.Env).To(ContainElement(credentialsEnvVar)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} + +func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { + // Check that the kube-controller-manager container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=external")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(ContainElement("--external-cloud-volume-plugin=gce")) + Expect(c.Env).To(ContainElement(credentialsEnvVar)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} + +func checkOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) { + u := controlplane.UnitWithName(osc.Spec.Units, "kubelet.service") + Expect(u).To(Not(BeNil())) + Expect(u.Content).To(Equal(util.StringPtr(newServiceContent))) + f := controlplane.FileWithPath(osc.Spec.Files, "/var/lib/kubelet/config/kubelet") + Expect(f).To(Not(BeNil())) + Expect(f.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData})) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go new file mode 100644 index 000000000..7b8f9c932 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go @@ -0,0 +1,40 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = log.Log.WithName("aws-controlplaneexposure-webhook") + +// AddToManager adds a webhook to the given manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.SeedKind, + Provider: gcp.Type, + Types: []runtime.Object{&appsv1.Deployment{}}, + Mutator: NewMutator(logger), + }) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator.go new file mode 100644 index 000000000..1039377c7 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator.go @@ -0,0 +1,71 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewMutator creates a new controlplaneexposure mutator. +func NewMutator(logger logr.Logger) controlplane.Mutator { + return &mutator{ + logger: logger.WithName("mutator"), + } +} + +type mutator struct { + client client.Client + logger logr.Logger +} + +// InjectClient injects the given client into the mutator. +func (m *mutator) InjectClient(client client.Client) error { + m.client = client + return nil +} + +// Mutate validates and if needed mutates the given object. +func (m *mutator) Mutate(ctx context.Context, obj runtime.Object) error { + switch x := obj.(type) { + case *appsv1.Deployment: + switch x.Name { + case common.KubeAPIServerDeploymentName: + // Get load balancer address of the kube-apiserver service + address, err := controlplane.GetLoadBalancerIngress(ctx, m.client, x.Namespace, common.KubeAPIServerDeploymentName) + if err != nil { + return errors.Wrap(err, "could not get kube-apiserver service load balancer address") + } + + return mutateKubeAPIServerDeployment(x, address) + } + } + return nil +} + +func mutateKubeAPIServerDeployment(dep *appsv1.Deployment, address string) error { + if c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver"); c != nil { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--advertise-address=", address) + } + return nil +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator_test.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator_test.go new file mode 100644 index 000000000..a7ccb57aa --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/mutator_test.go @@ -0,0 +1,154 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + "testing" + + mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "GCP Controlplane Exposure Webhook Suite") +} + +var _ = Describe("Mutator", func() { + var ( + ctrl *gomock.Controller + + svcKey = client.ObjectKey{Namespace: namespace, Name: common.KubeAPIServerDeploymentName} + svc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "1.2.3.4"}, + }, + }, + }, + } + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#Mutate", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create mutator + mutator := NewMutator(logger) + err := mutator.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call Mutate method and check the result + err = mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{"--advertise-address=?"}, + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create mutator + mutator := NewMutator(logger) + err := mutator.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call Mutate method and check the result + err = mutator.Mutate(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--advertise-address=1.2.3.4")) +} + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *corev1.Service: + *obj.(*corev1.Service) = *result.(*corev1.Service) + } + return nil + } +} diff --git a/controllers/provider-gcp/pkg/webhook/webhook.go b/controllers/provider-gcp/pkg/webhook/webhook.go new file mode 100644 index 000000000..d84f496f6 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/webhook.go @@ -0,0 +1,31 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplaneexposure" + "github.com/gardener/gardener-extensions/pkg/webhook" +) + +var ( + addToManagerBuilder = webhook.NewAddToManagerBuilder( + controlplaneexposure.AddToManager, + controlplane.AddToManager, + ) + + // AddToManager adds all provider webhooks to the given manager. + AddToManager = addToManagerBuilder.AddToManager +) diff --git a/pkg/mock/gardener-extensions/webhook/controlplane/doc.go b/pkg/mock/gardener-extensions/webhook/controlplane/doc.go index db6c85079..5b0bc2399 100644 --- a/pkg/mock/gardener-extensions/webhook/controlplane/doc.go +++ b/pkg/mock/gardener-extensions/webhook/controlplane/doc.go @@ -12,6 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate mockgen -package=controlplane -destination=mocks.go github.com/gardener/gardener-extensions/pkg/webhook/controlplane Mutator +//go:generate mockgen -package=controlplane -destination=mocks.go github.com/gardener/gardener-extensions/pkg/webhook/controlplane Mutator,KubeletConfigCodec,UnitSerializer package controlplane diff --git a/pkg/mock/gardener-extensions/webhook/controlplane/mocks.go b/pkg/mock/gardener-extensions/webhook/controlplane/mocks.go index cb1143f66..4e96786e6 100644 --- a/pkg/mock/gardener-extensions/webhook/controlplane/mocks.go +++ b/pkg/mock/gardener-extensions/webhook/controlplane/mocks.go @@ -1,13 +1,16 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/gardener/gardener-extensions/pkg/webhook/controlplane (interfaces: Mutator) +// Source: github.com/gardener/gardener-extensions/pkg/webhook/controlplane (interfaces: Mutator,KubeletConfigCodec,UnitSerializer) // Package controlplane is a generated GoMock package. package controlplane import ( context "context" + unit "github.com/coreos/go-systemd/unit" + v1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" gomock "github.com/golang/mock/gomock" runtime "k8s.io/apimachinery/pkg/runtime" + v1beta1 "k8s.io/kubelet/config/v1beta1" reflect "reflect" ) @@ -47,3 +50,109 @@ func (mr *MockMutatorMockRecorder) Mutate(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mutate", reflect.TypeOf((*MockMutator)(nil).Mutate), arg0, arg1) } + +// MockKubeletConfigCodec is a mock of KubeletConfigCodec interface +type MockKubeletConfigCodec struct { + ctrl *gomock.Controller + recorder *MockKubeletConfigCodecMockRecorder +} + +// MockKubeletConfigCodecMockRecorder is the mock recorder for MockKubeletConfigCodec +type MockKubeletConfigCodecMockRecorder struct { + mock *MockKubeletConfigCodec +} + +// NewMockKubeletConfigCodec creates a new mock instance +func NewMockKubeletConfigCodec(ctrl *gomock.Controller) *MockKubeletConfigCodec { + mock := &MockKubeletConfigCodec{ctrl: ctrl} + mock.recorder = &MockKubeletConfigCodecMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockKubeletConfigCodec) EXPECT() *MockKubeletConfigCodecMockRecorder { + return m.recorder +} + +// Decode mocks base method +func (m *MockKubeletConfigCodec) Decode(arg0 *v1alpha1.FileContentInline) (*v1beta1.KubeletConfiguration, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Decode", arg0) + ret0, _ := ret[0].(*v1beta1.KubeletConfiguration) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Decode indicates an expected call of Decode +func (mr *MockKubeletConfigCodecMockRecorder) Decode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Decode", reflect.TypeOf((*MockKubeletConfigCodec)(nil).Decode), arg0) +} + +// Encode mocks base method +func (m *MockKubeletConfigCodec) Encode(arg0 *v1beta1.KubeletConfiguration, arg1 string) (*v1alpha1.FileContentInline, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Encode", arg0, arg1) + ret0, _ := ret[0].(*v1alpha1.FileContentInline) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Encode indicates an expected call of Encode +func (mr *MockKubeletConfigCodecMockRecorder) Encode(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockKubeletConfigCodec)(nil).Encode), arg0, arg1) +} + +// MockUnitSerializer is a mock of UnitSerializer interface +type MockUnitSerializer struct { + ctrl *gomock.Controller + recorder *MockUnitSerializerMockRecorder +} + +// MockUnitSerializerMockRecorder is the mock recorder for MockUnitSerializer +type MockUnitSerializerMockRecorder struct { + mock *MockUnitSerializer +} + +// NewMockUnitSerializer creates a new mock instance +func NewMockUnitSerializer(ctrl *gomock.Controller) *MockUnitSerializer { + mock := &MockUnitSerializer{ctrl: ctrl} + mock.recorder = &MockUnitSerializerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockUnitSerializer) EXPECT() *MockUnitSerializerMockRecorder { + return m.recorder +} + +// Deserialize mocks base method +func (m *MockUnitSerializer) Deserialize(arg0 string) ([]*unit.UnitOption, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Deserialize", arg0) + ret0, _ := ret[0].([]*unit.UnitOption) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Deserialize indicates an expected call of Deserialize +func (mr *MockUnitSerializerMockRecorder) Deserialize(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Deserialize", reflect.TypeOf((*MockUnitSerializer)(nil).Deserialize), arg0) +} + +// Serialize mocks base method +func (m *MockUnitSerializer) Serialize(arg0 []*unit.UnitOption) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Serialize", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Serialize indicates an expected call of Serialize +func (mr *MockUnitSerializerMockRecorder) Serialize(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Serialize", reflect.TypeOf((*MockUnitSerializer)(nil).Serialize), arg0) +} diff --git a/pkg/util/pointers.go b/pkg/util/pointers.go index 3939fc34a..77476fd3a 100644 --- a/pkg/util/pointers.go +++ b/pkg/util/pointers.go @@ -23,3 +23,8 @@ func BoolPtr(b bool) *bool { func Int32Ptr(i int32) *int32 { return &i } + +// StringPtr returns a String pointer to its argument. +func StringPtr(s string) *string { + return &s +} diff --git a/pkg/webhook/controlplane/gardener.go b/pkg/webhook/controlplane/gardener.go new file mode 100644 index 000000000..3e7de6e17 --- /dev/null +++ b/pkg/webhook/controlplane/gardener.go @@ -0,0 +1,51 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetLoadBalancerIngress takes a context, a client, a namespace and a service name. It queries for a load balancer's technical name +// (ip address or hostname). It returns the value of the technical name whereby it always prefers the IP address (if given) +// over the hostname. It also returns the list of all load balancer ingresses. +// TODO This function is copy / pasted from Gardener, remove it once dependency to Gardener is updated +func GetLoadBalancerIngress(ctx context.Context, client client.Client, namespace, name string) (string, error) { + service := &corev1.Service{} + if err := client.Get(ctx, kutil.Key(namespace, name), service); err != nil { + return "", err + } + + var ( + serviceStatusIngress = service.Status.LoadBalancer.Ingress + length = len(serviceStatusIngress) + ) + + switch { + case length == 0: + return "", errors.New("`.status.loadBalancer.ingress[]` has no elements yet, i.e. external load balancer has not been created (is your quota limit exceeded/reached?)") + case serviceStatusIngress[length-1].IP != "": + return serviceStatusIngress[length-1].IP, nil + case serviceStatusIngress[length-1].Hostname != "": + return serviceStatusIngress[length-1].Hostname, nil + } + + return "", errors.New("`.status.loadBalancer.ingress[]` has an element which does neither contain `.ip` nor `.hostname`") +} diff --git a/pkg/webhook/controlplane/handler.go b/pkg/webhook/controlplane/handler.go index c43c40b65..c2c5267b6 100644 --- a/pkg/webhook/controlplane/handler.go +++ b/pkg/webhook/controlplane/handler.go @@ -24,8 +24,10 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "sigs.k8s.io/controller-runtime/pkg/webhook/admission/types" ) @@ -53,12 +55,21 @@ type handler struct { logger logr.Logger } -// InjectDecoder injects the decoder into the handler. +// InjectDecoder injects the given decoder into the handler. func (h *handler) InjectDecoder(d types.Decoder) error { h.decoder = d return nil } +// InjectClient injects the given client into the mutator. +// TODO Replace this with the more generic InjectFunc when controller runtime supports it +func (h *handler) InjectClient(client client.Client) error { + if _, err := inject.ClientInto(client, h.mutator); err != nil { + return errors.Wrap(err, "could not inject the client into the mutator") + } + return nil +} + // Handle handles the given admission request. func (h *handler) Handle(ctx context.Context, req types.Request) types.Response { ar := req.AdmissionRequest diff --git a/pkg/webhook/controlplane/utils.go b/pkg/webhook/controlplane/utils.go index c7647f938..5cff5fa54 100644 --- a/pkg/webhook/controlplane/utils.go +++ b/pkg/webhook/controlplane/utils.go @@ -179,6 +179,14 @@ func EnsureNoVolumeWithName(items []corev1.Volume, name string) []corev1.Volume return items } +// EnsureUnitOption ensures the given unit option exist in the given slice. +func EnsureUnitOption(items []*unit.UnitOption, item *unit.UnitOption) []*unit.UnitOption { + if i := unitOptionIndex(items, item); i < 0 { + items = append(items, item) + } + return items +} + // StringIndex returns the index of the first occurrence of the given string in the given slice, or -1 if not found. func StringIndex(items []string, value string) int { for i, item := range items { @@ -235,6 +243,15 @@ func unitOptionWithSectionAndNameIndex(items []*unit.UnitOption, section, name s return -1 } +func unitOptionIndex(items []*unit.UnitOption, item *unit.UnitOption) int { + for i := range items { + if reflect.DeepEqual(items[i], item) { + return i + } + } + return -1 +} + func envVarWithNameIndex(items []corev1.EnvVar, name string) int { for i, item := range items { if item.Name == name {