Merge pull request #79 from stoyanr/add-gcp-controlplane-webhooks

Add GCP controlplane webhooks

Makefile

@@ -125,6 +125,9 @@ start-provider-gcp:
 		-ldflags $(LD_FLAGS) \
 		./controllers/provider-gcp/cmd/gardener-extension-provider-gcp \
 		--leader-election=$(LEADER_ELECTION) \
+		--webhook-config-mode=url \
+		--webhook-config-name=gcp-webhooks \
+		--webhook-config-host=$(HOSTNAME) \
 		--infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION)
 
 .PHONY: start-provider-openstack

controllers/provider-aws/pkg/webhook/controlplane/add.go

@@ -36,6 +36,6 @@ func AddToManager(mgr manager.Manager) (webhook.Webhook, error) {
 		Kind:     extensionswebhook.ShootKind,
 		Provider: aws.Type,
 		Types:    []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}},
-		Mutator:  newMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger),
+		Mutator:  NewMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger),
 	})
 }

controllers/provider-aws/pkg/webhook/controlplane/mutator.go

@@ -31,8 +31,8 @@ import (
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 )
 
-// newMutator creates a new controlplane mutator.
-func newMutator(unitSerializer controlplane.UnitSerializer, kubeletConfigCodec controlplane.KubeletConfigCodec, logger logr.Logger) *mutator {
+// NewMutator creates a new controlplane mutator.
+func NewMutator(unitSerializer controlplane.UnitSerializer, kubeletConfigCodec controlplane.KubeletConfigCodec, logger logr.Logger) controlplane.Mutator {
 	return &mutator{
 		unitSerializer:     unitSerializer,
 		kubeletConfigCodec: kubeletConfigCodec,

controllers/provider-aws/pkg/webhook/controlplane/mutator_test.go

@@ -19,15 +19,29 @@ import (
 	"testing"
 
 	"github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws"
+	mockcontrolplane "github.com/gardener/gardener-extensions/pkg/mock/gardener-extensions/webhook/controlplane"
+	"github.com/gardener/gardener-extensions/pkg/util"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test"
 
+	"github.com/coreos/go-systemd/unit"
+	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	"github.com/gardener/gardener/pkg/operation/common"
+	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+)
+
+const (
+	oldServiceContent = "old kubelet.service content"
+	newServiceContent = "new kubelet.service content"
+
+	oldKubeletConfigData = "old kubelet config data"
+	newKubeletConfigData = "new kubelet config data"
 )
 
 func TestController(t *testing.T) {
@@ -36,6 +50,16 @@ func TestController(t *testing.T) {
 }
 
 var _ = Describe("Mutator", func() {
+	var (
+		ctrl *gomock.Controller
+	)
+
+	BeforeEach(func() {
+		ctrl = gomock.NewController(GinkgoT())
+	})
+	AfterEach(func() {
+		ctrl.Finish()
+	})
 
 	Describe("#Mutate", func() {
 		It("should add missing elements to kube-apiserver deployment", func() {
@@ -56,7 +80,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(nil, nil, logger)
+			// Create mutator
+			mutator := NewMutator(nil, nil, logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeAPIServerDeployment(dep)
@@ -100,7 +127,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(nil, nil, logger)
+			// Create mutator
+			mutator := NewMutator(nil, nil, logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeAPIServerDeployment(dep)
@@ -124,7 +154,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(nil, nil, logger)
+			// Create mutator
+			mutator := NewMutator(nil, nil, logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeControllerManagerDeployment(dep)
@@ -165,11 +198,90 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(nil, nil, logger)
+			// Create mutator
+			mutator := NewMutator(nil, nil, logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeControllerManagerDeployment(dep)
 		})
+
+		It("should modify existing elements of OperatingSystemConfig", func() {
+			var (
+				osc = &extensionsv1alpha1.OperatingSystemConfig{
+					ObjectMeta: metav1.ObjectMeta{Name: "test"},
+					Spec: extensionsv1alpha1.OperatingSystemConfigSpec{
+						Purpose: extensionsv1alpha1.OperatingSystemConfigPurposeReconcile,
+						Units: []extensionsv1alpha1.Unit{
+							{
+								Name:    "kubelet.service",
+								Content: util.StringPtr(oldServiceContent),
+							},
+						},
+						Files: []extensionsv1alpha1.File{
+							{
+								Path: "/var/lib/kubelet/config/kubelet",
+								Content: extensionsv1alpha1.FileContent{
+									Inline: &extensionsv1alpha1.FileContentInline{
+										Data: oldKubeletConfigData,
+									},
+								},
+							},
+						},
+					},
+				}
+
+				oldUnitOptions = []*unit.UnitOption{
+					{
+						Section: "Service",
+						Name:    "ExecStart",
+						Value: `/opt/bin/hyperkube kubelet \
+    --config=/var/lib/kubelet/config/kubelet`,
+					},
+				}
+				newUnitOptions = []*unit.UnitOption{
+					{
+						Section: "Service",
+						Name:    "ExecStart",
+						Value: `/opt/bin/hyperkube kubelet \
+    --config=/var/lib/kubelet/config/kubelet \
+    --cloud-provider=aws`,
+					},
+				}
+
+				oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{
+					FeatureGates: map[string]bool{
+						"Foo":                      true,
+						"VolumeSnapshotDataSource": true,
+						"CSINodeInfo":              true,
+					},
+				}
+				newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{
+					FeatureGates: map[string]bool{
+						"Foo": true,
+					},
+				}
+			)
+
+			// Create mock UnitSerializer
+			us := mockcontrolplane.NewMockUnitSerializer(ctrl)
+			us.EXPECT().Deserialize(oldServiceContent).Return(oldUnitOptions, nil)
+			us.EXPECT().Serialize(newUnitOptions).Return(newServiceContent, nil)
+
+			// Create mock KubeletConfigCodec
+			kcc := mockcontrolplane.NewMockKubeletConfigCodec(ctrl)
+			kcc.EXPECT().Decode(&extensionsv1alpha1.FileContentInline{Data: oldKubeletConfigData}).Return(oldKubeletConfig, nil)
+			kcc.EXPECT().Encode(newKubeletConfig, "").Return(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData}, nil)
+
+			// Create mutator
+			mutator := NewMutator(us, kcc, logger)
+
+			// Call Mutate method and check the result
+			err := mutator.Mutate(context.TODO(), osc)
+			Expect(err).To(Not(HaveOccurred()))
+			checkOperatingSystemConfig(osc)
+		})
 	})
 })
 
@@ -209,3 +321,12 @@ func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) {
 	Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume))
 	Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume))
 }
+
+func checkOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) {
+	u := controlplane.UnitWithName(osc.Spec.Units, "kubelet.service")
+	Expect(u).To(Not(BeNil()))
+	Expect(u.Content).To(Equal(util.StringPtr(newServiceContent)))
+	f := controlplane.FileWithPath(osc.Spec.Files, "/var/lib/kubelet/config/kubelet")
+	Expect(f).To(Not(BeNil()))
+	Expect(f.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData}))
+}

controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go

@@ -36,6 +36,6 @@ func AddToManager(mgr manager.Manager) (webhook.Webhook, error) {
 		Kind:     extensionswebhook.SeedKind,
 		Provider: aws.Type,
 		Types:    []runtime.Object{&corev1.Service{}, &appsv1.Deployment{}},
-		Mutator:  newMutator(logger),
+		Mutator:  NewMutator(logger),
 	})
 }

controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator.go

@@ -26,8 +26,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-// newMutator creates a new controlplaneexposure mutator.
-func newMutator(logger logr.Logger) *mutator {
+// NewMutator creates a new controlplaneexposure mutator.
+func NewMutator(logger logr.Logger) controlplane.Mutator {
 	return &mutator{
 		logger: logger.WithName("mutator"),
 	}
@@ -38,7 +38,7 @@ type mutator struct {
 }
 
 // Mutate validates and if needed mutates the given object.
-func (v *mutator) Mutate(ctx context.Context, obj runtime.Object) error {
+func (m *mutator) Mutate(ctx context.Context, obj runtime.Object) error {
 	switch x := obj.(type) {
 	case *corev1.Service:
 		switch x.Name {

controllers/provider-aws/pkg/webhook/controlplaneexposure/mutator_test.go

@@ -43,7 +43,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(logger)
+			// Create mutator
+			mutator := NewMutator(logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), svc)
 			Expect(err).To(Not(HaveOccurred()))
 			Expect(svc.Annotations).To(HaveKeyWithValue("service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout", "3600"))
@@ -63,7 +66,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(logger)
+			// Create mutator
+			mutator := NewMutator(logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), svc)
 			Expect(err).To(Not(HaveOccurred()))
 			Expect(svc.Annotations).To(BeEmpty())
@@ -87,7 +93,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(logger)
+			// Create mutator
+			mutator := NewMutator(logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeAPIServerDeployment(dep)
@@ -112,7 +121,10 @@ var _ = Describe("Mutator", func() {
 				}
 			)
 
-			mutator := newMutator(logger)
+			// Create mutator
+			mutator := NewMutator(logger)
+
+			// Call Mutate method and check the result
 			err := mutator.Mutate(context.TODO(), dep)
 			Expect(err).To(Not(HaveOccurred()))
 			checkKubeAPIServerDeployment(dep)

controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go

@@ -23,9 +23,11 @@ import (
 	gcpcontroller "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller"
 	gcpcontrolplane "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/controlplane"
 	gcpinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/infrastructure"
+	gcpwebhook "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook"
 	"github.com/gardener/gardener-extensions/pkg/controller"
 	controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd"
 	"github.com/gardener/gardener-extensions/pkg/controller/infrastructure"
+	webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd"
 
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -58,7 +60,17 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 		}
 		controlPlaneOpts = controllercmd.PrefixOption("controlplane-", controlPlaneCtrlOpts)
 
-		aggOption = controllercmd.NewOptionAggregator(restOpts, mgrOpts, infraOpts, controlPlaneOpts)
+		webhookServerOpts = &webhookcmd.WebhookServerOptions{
+			Port:             7890,
+			CertDir:          "/tmp/cert",
+			Mode:             webhookcmd.ServiceMode,
+			Name:             "webhooks",
+			Namespace:        os.Getenv("WEBHOOK_CONFIG_NAMESPACE"),
+			ServiceSelectors: "{}",
+			Host:             "localhost",
+		}
+
+		aggOption = controllercmd.NewOptionAggregator(restOpts, mgrOpts, infraOpts, controlPlaneOpts, webhookServerOpts)
 	)
 
 	cmd := &cobra.Command{
@@ -90,6 +102,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 				controllercmd.LogErrAndExit(err, "Could not add controllers to manager")
 			}
 
+			if err := gcpwebhook.AddToManager(mgr, webhookServerOpts.Completed()); err != nil {
+				controllercmd.LogErrAndExit(err, "Could not add webhooks to manager")
+			}
+
 			if err := mgr.Start(ctx.Done()); err != nil {
 				controllercmd.LogErrAndExit(err, "Error running manager")
 			}

controllers/provider-gcp/pkg/webhook/controlplane/add.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp"
+	extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook"
+	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
+
+	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+var logger = log.Log.WithName("gcp-controlplane-webhook")
+
+// AddToManager adds a webhook to the given manager.
+func AddToManager(mgr manager.Manager) (webhook.Webhook, error) {
+	logger.Info("Adding webhook to manager")
+	return controlplane.Add(mgr, controlplane.AddArgs{
+		Kind:     extensionswebhook.ShootKind,
+		Provider: gcp.Type,
+		Types:    []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}},
+		Mutator:  NewMutator(controlplane.NewUnitSerializer(), controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger),
+	})
+}