[WIP] Ensure Cloud provider config content as part of kubelet configu…

…ration

controllers/provider-azure/pkg/webhook/controlplane/ensurer.go

@@ -16,12 +16,15 @@ package controlplane
 
 import (
 	"context"
+	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack"
+	"github.com/pkg/errors"
 
 	"github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/azure"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator"
 
 	"github.com/coreos/go-systemd/unit"
+	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -143,3 +146,27 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig
 	delete(kubeletConfig.FeatureGates, "CSIDriverRegistry")
 	return nil
 }
+
+//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration.
+func (e *ensurer) ShouldProvisionKubeletCloudProviderConfig() bool {
+	return true
+}
+
+//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file conforms to the provider requirements.
+func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, data *string, namespace string) error {
+	// Get `cloud-provider-config` ConfigMap
+	var cm corev1.ConfigMap
+	err := e.client.Get(ctx, kutil.Key(namespace, openstack.CloudProviderConfigName), &cm)
+	if err != nil {
+		return errors.Wrapf(err, "could not get configmap with name '%s' and namespace '%s'", openstack.CloudProviderConfigName, namespace)
+	}
+
+	// Check if the `cloud-provider-config` has "cloudprovider.conf" key
+	if cm.Data == nil || cm.Data["cloudprovider.conf"] == "" {
+		return nil
+	}
+
+	// Overwrite data variable
+	*data = cm.Data["cloudprovider.conf"]
+	return nil
+}

controllers/provider-azure/pkg/webhook/controlplane/ensurer_test.go

@@ -16,6 +16,7 @@ package controlplane
 
 import (
 	"context"
+	"github.com/gardener/gardener-extensions/pkg/util"
 	"testing"
 
 	"github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/azure"
@@ -38,7 +39,8 @@ import (
 )
 
 const (
-	namespace = "test"
+	namespace                  = "test"
+	cloudProviderConfigContent = "[Global]\nauth-url: https://cluster.eu-de-200.cloud.sap:5000/v3/\n"
 )
 
 func TestController(t *testing.T) {
@@ -53,11 +55,11 @@ var _ = Describe("Ensurer", func() {
 		cmKey = client.ObjectKey{Namespace: namespace, Name: azure.CloudProviderConfigName}
 		cm    = &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: azure.CloudProviderConfigName},
-			Data:       map[string]string{"abc": "xyz"},
+			Data:       map[string]string{"abc": "xyz", "cloudprovider.conf": cloudProviderConfigContent},
 		}
 
 		annotations = map[string]string{
-			"checksum/configmap-" + azure.CloudProviderConfigName: "08a7bc7fe8f59b055f173145e211760a83f02cf89635cef26ebb351378635606",
+			"checksum/configmap-" + azure.CloudProviderConfigName: "2ac8b96caad089f7b0217f0b2916ff4e8d4346655746de55178207e180cf0bbe",
 		}
 	)
 
@@ -287,6 +289,43 @@ var _ = Describe("Ensurer", func() {
 			Expect(&kubeletConfig).To(Equal(newKubeletConfig))
 		})
 	})
+
+	Describe("#EnsureKubeletCloudProviderConfig", func() {
+		var (
+			existingData = util.StringPtr("[LoadBalancer]\nlb-version=v2\nlb-provider:\n")
+			emptydata    = util.StringPtr("")
+		)
+		It("should create element containing cloud provider config content", func() {
+			// Create mock client
+			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			// Create ensurer
+			ensurer := NewEnsurer(logger)
+			err := ensurer.(inject.Client).InjectClient(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Call EnsureKubeletConfiguration method and check the result
+			err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), emptydata, namespace)
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(*emptydata).To(Equal(cloudProviderConfigContent))
+		})
+		It("should modify existing element containing cloud provider config content", func() {
+			// Create mock client
+			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			// Create ensurer
+			ensurer := NewEnsurer(logger)
+			err := ensurer.(inject.Client).InjectClient(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Call EnsureKubeletConfiguration method and check the result
+			err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), existingData, namespace)
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(*existingData).To(Equal(cloudProviderConfigContent))
+		})
+	})
 })
 
 func checkKubeAPIServerDeployment(dep *appsv1.Deployment, annotations map[string]string) {

controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go

@@ -16,12 +16,13 @@ package controlplane
 
 import (
 	"context"
-
 	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator"
+	"github.com/pkg/errors"
 
 	"github.com/coreos/go-systemd/unit"
+	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -126,11 +127,18 @@ func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, opts []*u
 		command = ensureKubeletCommandLineArgs(command)
 		opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n    ")
 	}
+
+	opts = controlplane.EnsureUnitOption(opts, &unit.UnitOption{
+		Section: "Service",
+		Name:    "ExecStartPre",
+		Value:   `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`,
+	})
 	return opts, nil
 }
 
 func ensureKubeletCommandLineArgs(command []string) []string {
 	command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "openstack")
+	command = controlplane.EnsureStringWithPrefix(command, "--cloud-config=", "/var/lib/kubelet/cloudprovider.conf")
 	return command
 }
 
@@ -143,3 +151,27 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig
 	delete(kubeletConfig.FeatureGates, "CSIDriverRegistry")
 	return nil
 }
+
+//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration.
+func (e *ensurer) ShouldProvisionKubeletCloudProviderConfig() bool {
+	return true
+}
+
+//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file conforms to the provider requirements.
+func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, data *string, namespace string) error {
+	// Get `cloud-provider-config` ConfigMap
+	var cm corev1.ConfigMap
+	err := e.client.Get(ctx, kutil.Key(namespace, openstack.CloudProviderConfigName), &cm)
+	if err != nil {
+		return errors.Wrapf(err, "could not get configmap with name '%s' and namespace '%s'", openstack.CloudProviderConfigName, namespace)
+	}
+
+	// Check if the `cloud-provider-config` has "cloudprovider.conf" key
+	if cm.Data == nil || cm.Data["cloudprovider.conf"] == "" {
+		return nil
+	}
+
+	// Overwrite data variable
+	*data = cm.Data["cloudprovider.conf"]
+	return nil
+}

controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go

@@ -16,6 +16,7 @@ package controlplane
 
 import (
 	"context"
+	"github.com/gardener/gardener-extensions/pkg/util"
 	"testing"
 
 	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack"
@@ -38,7 +39,8 @@ import (
 )
 
 const (
-	namespace = "test"
+	namespace                  = "test"
+	cloudProviderConfigContent = "[Global]\nauth-url: https://cluster.eu-de-200.cloud.sap:5000/v3/\n"
 )
 
 func TestController(t *testing.T) {
@@ -53,11 +55,11 @@ var _ = Describe("Ensurer", func() {
 		cmKey = client.ObjectKey{Namespace: namespace, Name: openstack.CloudProviderConfigName}
 		cm    = &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: openstack.CloudProviderConfigName},
-			Data:       map[string]string{"abc": "xyz"},
+			Data:       map[string]string{"abc": "xyz", "cloudprovider.conf": cloudProviderConfigContent},
 		}
 
 		annotations = map[string]string{
-			"checksum/configmap-" + openstack.CloudProviderConfigName: "08a7bc7fe8f59b055f173145e211760a83f02cf89635cef26ebb351378635606",
+			"checksum/configmap-" + openstack.CloudProviderConfigName: "2ac8b96caad089f7b0217f0b2916ff4e8d4346655746de55178207e180cf0bbe",
 		}
 	)
 
@@ -245,7 +247,13 @@ var _ = Describe("Ensurer", func() {
 						Name:    "ExecStart",
 						Value: `/opt/bin/hyperkube kubelet \
     --config=/var/lib/kubelet/config/kubelet \
-    --cloud-provider=openstack`,
+    --cloud-provider=openstack \
+    --cloud-config=/var/lib/kubelet/cloudprovider.conf`,
+					},
+					{
+						Section: "Service",
+						Name:    "ExecStartPre",
+						Value:   `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`,
 					},
 				}
 			)
@@ -287,6 +295,43 @@ var _ = Describe("Ensurer", func() {
 			Expect(&kubeletConfig).To(Equal(newKubeletConfig))
 		})
 	})
+
+	Describe("#EnsureKubeletCloudProviderConfig", func() {
+		var (
+			existingData = util.StringPtr("[LoadBalancer]\nlb-version=v2\nlb-provider:\n")
+			emptydata    = util.StringPtr("")
+		)
+		It("should create element containing cloud provider config content", func() {
+			// Create mock client
+			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			// Create ensurer
+			ensurer := NewEnsurer(logger)
+			err := ensurer.(inject.Client).InjectClient(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Call EnsureKubeletConfiguration method and check the result
+			err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), emptydata, namespace)
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(*emptydata).To(Equal(cloudProviderConfigContent))
+		})
+		It("should modify existing element containing cloud provider config content", func() {
+			// Create mock client
+			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			// Create ensurer
+			ensurer := NewEnsurer(logger)
+			err := ensurer.(inject.Client).InjectClient(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Call EnsureKubeletConfiguration method and check the result
+			err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), existingData, namespace)
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(*existingData).To(Equal(cloudProviderConfigContent))
+		})
+	})
 })
 
 func checkKubeAPIServerDeployment(dep *appsv1.Deployment, annotations map[string]string) {

pkg/webhook/controlplane/genericmutator/mutator.go

@@ -17,6 +17,7 @@ package genericmutator
 import (
 	"context"
 	extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller"
+	"github.com/gardener/gardener-extensions/pkg/util"
 	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
 
 	"github.com/coreos/go-systemd/unit"
@@ -51,6 +52,10 @@ type Ensurer interface {
 	EnsureKubeletConfiguration(context.Context, *kubeletconfigv1beta1.KubeletConfiguration) error
 	// EnsureKubernetesGeneralConfiguration ensures that the kubernetes general configuration conforms to the provider requirements.
 	EnsureKubernetesGeneralConfiguration(context.Context, *string) error
+	//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration.
+	ShouldProvisionKubeletCloudProviderConfig() bool
+	//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file content conforms to the provider requirements.
+	EnsureKubeletCloudProviderConfig(context.Context, *string, string) error
 }
 
 // NewMutator creates a new controlplane mutator.
@@ -139,6 +144,13 @@ func (m *mutator) mutateOperatingSystemConfig(ctx context.Context, osc *extensio
 		}
 	}
 
+	// Check if cloudprovider.conf needs to be ensured
+	if m.ensurer.ShouldProvisionKubeletCloudProviderConfig() {
+		if err := m.ensureKubeletCloudProviderConfig(ctx, osc); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -193,3 +205,40 @@ func (m *mutator) ensureKubernetesGeneralConfigFileContent(ctx context.Context,
 
 	return nil
 }
+
+const cloudProviderConfigPath = "/var/lib/kubelet/cloudprovider.conf"
+
+func (m *mutator) ensureKubeletCloudProviderConfig(ctx context.Context, osc *extensionsv1alpha1.OperatingSystemConfig) error {
+	// Create file if it does not exist
+	f := controlplane.FileWithPath(osc.Spec.Files, cloudProviderConfigPath)
+	if f == nil {
+		f = &extensionsv1alpha1.File{
+			Path: cloudProviderConfigPath,
+		}
+		osc.Spec.Files = append(osc.Spec.Files, *f)
+	}
+
+	// Ensure permissions and content inline encoding
+	f = controlplane.FileWithPath(osc.Spec.Files, cloudProviderConfigPath)
+	f.Permissions = util.Int32Ptr(0644)
+	f.Content = extensionsv1alpha1.FileContent{
+		Inline: &extensionsv1alpha1.FileContentInline{
+			Encoding: "b64",
+		},
+	}
+
+	// Ensure content inline data
+	data := f.Content.Inline.Data
+	err := m.ensurer.EnsureKubeletCloudProviderConfig(ctx, &data, osc.Namespace)
+	if err != nil {
+		return err
+	}
+	f.Content.Inline.Data = data
+
+	err = m.client.Update(ctx, osc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}