diff --git a/controllers/provider-azure/pkg/webhook/controlplane/ensurer.go b/controllers/provider-azure/pkg/webhook/controlplane/ensurer.go index fad57feac..5c1d93433 100644 --- a/controllers/provider-azure/pkg/webhook/controlplane/ensurer.go +++ b/controllers/provider-azure/pkg/webhook/controlplane/ensurer.go @@ -16,12 +16,15 @@ package controlplane import ( "context" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + "github.com/pkg/errors" "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/azure" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" "github.com/coreos/go-systemd/unit" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -143,3 +146,27 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig delete(kubeletConfig.FeatureGates, "CSIDriverRegistry") return nil } + +//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration. +func (e *ensurer) ShouldProvisionKubeletCloudProviderConfig() bool { + return true +} + +//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file conforms to the provider requirements. +func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, data *string, namespace string) error { + // Get `cloud-provider-config` ConfigMap + var cm corev1.ConfigMap + err := e.client.Get(ctx, kutil.Key(namespace, openstack.CloudProviderConfigName), &cm) + if err != nil { + return errors.Wrapf(err, "could not get configmap with name '%s' and namespace '%s'", openstack.CloudProviderConfigName, namespace) + } + + // Check if the `cloud-provider-config` has "cloudprovider.conf" key + if cm.Data == nil || cm.Data["cloudprovider.conf"] == "" { + return nil + } + + // Overwrite data variable + *data = cm.Data["cloudprovider.conf"] + return nil +} diff --git a/controllers/provider-azure/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-azure/pkg/webhook/controlplane/ensurer_test.go index d9cf487f8..24372dbbd 100644 --- a/controllers/provider-azure/pkg/webhook/controlplane/ensurer_test.go +++ b/controllers/provider-azure/pkg/webhook/controlplane/ensurer_test.go @@ -16,6 +16,7 @@ package controlplane import ( "context" + "github.com/gardener/gardener-extensions/pkg/util" "testing" "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/azure" @@ -38,7 +39,8 @@ import ( ) const ( - namespace = "test" + namespace = "test" + cloudProviderConfigContent = "[Global]\nauth-url: https://cluster.eu-de-200.cloud.sap:5000/v3/\n" ) func TestController(t *testing.T) { @@ -53,11 +55,11 @@ var _ = Describe("Ensurer", func() { cmKey = client.ObjectKey{Namespace: namespace, Name: azure.CloudProviderConfigName} cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: azure.CloudProviderConfigName}, - Data: map[string]string{"abc": "xyz"}, + Data: map[string]string{"abc": "xyz", "cloudprovider.conf": cloudProviderConfigContent}, } annotations = map[string]string{ - "checksum/configmap-" + azure.CloudProviderConfigName: "08a7bc7fe8f59b055f173145e211760a83f02cf89635cef26ebb351378635606", + "checksum/configmap-" + azure.CloudProviderConfigName: "2ac8b96caad089f7b0217f0b2916ff4e8d4346655746de55178207e180cf0bbe", } ) @@ -287,6 +289,43 @@ var _ = Describe("Ensurer", func() { Expect(&kubeletConfig).To(Equal(newKubeletConfig)) }) }) + + Describe("#EnsureKubeletCloudProviderConfig", func() { + var ( + existingData = util.StringPtr("[LoadBalancer]\nlb-version=v2\nlb-provider:\n") + emptydata = util.StringPtr("") + ) + It("should create element containing cloud provider config content", func() { + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).NotTo(HaveOccurred()) + + // Call EnsureKubeletConfiguration method and check the result + err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), emptydata, namespace) + Expect(err).To(Not(HaveOccurred())) + Expect(*emptydata).To(Equal(cloudProviderConfigContent)) + }) + It("should modify existing element containing cloud provider config content", func() { + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).NotTo(HaveOccurred()) + + // Call EnsureKubeletConfiguration method and check the result + err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), existingData, namespace) + Expect(err).To(Not(HaveOccurred())) + Expect(*existingData).To(Equal(cloudProviderConfigContent)) + }) + }) }) func checkKubeAPIServerDeployment(dep *appsv1.Deployment, annotations map[string]string) { diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go index 33df9b1ed..942eadbbe 100644 --- a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go @@ -16,12 +16,13 @@ package controlplane import ( "context" - "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + "github.com/pkg/errors" "github.com/coreos/go-systemd/unit" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -126,11 +127,18 @@ func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, opts []*u command = ensureKubeletCommandLineArgs(command) opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n ") } + + opts = controlplane.EnsureUnitOption(opts, &unit.UnitOption{ + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`, + }) return opts, nil } func ensureKubeletCommandLineArgs(command []string) []string { command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "openstack") + command = controlplane.EnsureStringWithPrefix(command, "--cloud-config=", "/var/lib/kubelet/cloudprovider.conf") return command } @@ -143,3 +151,27 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig delete(kubeletConfig.FeatureGates, "CSIDriverRegistry") return nil } + +//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration. +func (e *ensurer) ShouldProvisionKubeletCloudProviderConfig() bool { + return true +} + +//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file conforms to the provider requirements. +func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, data *string, namespace string) error { + // Get `cloud-provider-config` ConfigMap + var cm corev1.ConfigMap + err := e.client.Get(ctx, kutil.Key(namespace, openstack.CloudProviderConfigName), &cm) + if err != nil { + return errors.Wrapf(err, "could not get configmap with name '%s' and namespace '%s'", openstack.CloudProviderConfigName, namespace) + } + + // Check if the `cloud-provider-config` has "cloudprovider.conf" key + if cm.Data == nil || cm.Data["cloudprovider.conf"] == "" { + return nil + } + + // Overwrite data variable + *data = cm.Data["cloudprovider.conf"] + return nil +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go index 590446b2e..77d741d1f 100644 --- a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go @@ -16,6 +16,7 @@ package controlplane import ( "context" + "github.com/gardener/gardener-extensions/pkg/util" "testing" "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" @@ -38,7 +39,8 @@ import ( ) const ( - namespace = "test" + namespace = "test" + cloudProviderConfigContent = "[Global]\nauth-url: https://cluster.eu-de-200.cloud.sap:5000/v3/\n" ) func TestController(t *testing.T) { @@ -53,11 +55,11 @@ var _ = Describe("Ensurer", func() { cmKey = client.ObjectKey{Namespace: namespace, Name: openstack.CloudProviderConfigName} cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: openstack.CloudProviderConfigName}, - Data: map[string]string{"abc": "xyz"}, + Data: map[string]string{"abc": "xyz", "cloudprovider.conf": cloudProviderConfigContent}, } annotations = map[string]string{ - "checksum/configmap-" + openstack.CloudProviderConfigName: "08a7bc7fe8f59b055f173145e211760a83f02cf89635cef26ebb351378635606", + "checksum/configmap-" + openstack.CloudProviderConfigName: "2ac8b96caad089f7b0217f0b2916ff4e8d4346655746de55178207e180cf0bbe", } ) @@ -245,7 +247,13 @@ var _ = Describe("Ensurer", func() { Name: "ExecStart", Value: `/opt/bin/hyperkube kubelet \ --config=/var/lib/kubelet/config/kubelet \ - --cloud-provider=openstack`, + --cloud-provider=openstack \ + --cloud-config=/var/lib/kubelet/cloudprovider.conf`, + }, + { + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`, }, } ) @@ -287,6 +295,43 @@ var _ = Describe("Ensurer", func() { Expect(&kubeletConfig).To(Equal(newKubeletConfig)) }) }) + + Describe("#EnsureKubeletCloudProviderConfig", func() { + var ( + existingData = util.StringPtr("[LoadBalancer]\nlb-version=v2\nlb-provider:\n") + emptydata = util.StringPtr("") + ) + It("should create element containing cloud provider config content", func() { + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).NotTo(HaveOccurred()) + + // Call EnsureKubeletConfiguration method and check the result + err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), emptydata, namespace) + Expect(err).To(Not(HaveOccurred())) + Expect(*emptydata).To(Equal(cloudProviderConfigContent)) + }) + It("should modify existing element containing cloud provider config content", func() { + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cmKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).NotTo(HaveOccurred()) + + // Call EnsureKubeletConfiguration method and check the result + err = ensurer.EnsureKubeletCloudProviderConfig(context.TODO(), existingData, namespace) + Expect(err).To(Not(HaveOccurred())) + Expect(*existingData).To(Equal(cloudProviderConfigContent)) + }) + }) }) func checkKubeAPIServerDeployment(dep *appsv1.Deployment, annotations map[string]string) { diff --git a/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go b/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go index 947fa8c1d..8257afe3f 100644 --- a/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go +++ b/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go @@ -108,6 +108,20 @@ func (mr *MockEnsurerMockRecorder) EnsureKubeSchedulerDeployment(arg0, arg1 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureKubeSchedulerDeployment", reflect.TypeOf((*MockEnsurer)(nil).EnsureKubeSchedulerDeployment), arg0, arg1) } +// EnsureKubeletCloudProviderConfig mocks base method +func (m *MockEnsurer) EnsureKubeletCloudProviderConfig(arg0 context.Context, arg1 *string, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureKubeletCloudProviderConfig", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureKubeletCloudProviderConfig indicates an expected call of EnsureKubeletCloudProviderConfig +func (mr *MockEnsurerMockRecorder) EnsureKubeletCloudProviderConfig(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureKubeletCloudProviderConfig", reflect.TypeOf((*MockEnsurer)(nil).EnsureKubeletCloudProviderConfig), arg0, arg1, arg2) +} + // EnsureKubeletConfiguration mocks base method func (m *MockEnsurer) EnsureKubeletConfiguration(arg0 context.Context, arg1 *v1beta1.KubeletConfiguration) error { m.ctrl.T.Helper() @@ -150,3 +164,17 @@ func (mr *MockEnsurerMockRecorder) EnsureKubernetesGeneralConfiguration(arg0, ar mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureKubernetesGeneralConfiguration", reflect.TypeOf((*MockEnsurer)(nil).EnsureKubernetesGeneralConfiguration), arg0, arg1) } + +// ShouldProvisionKubeletCloudProviderConfig mocks base method +func (m *MockEnsurer) ShouldProvisionKubeletCloudProviderConfig() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ShouldProvisionKubeletCloudProviderConfig") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ShouldProvisionKubeletCloudProviderConfig indicates an expected call of ShouldProvisionKubeletCloudProviderConfig +func (mr *MockEnsurerMockRecorder) ShouldProvisionKubeletCloudProviderConfig() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShouldProvisionKubeletCloudProviderConfig", reflect.TypeOf((*MockEnsurer)(nil).ShouldProvisionKubeletCloudProviderConfig)) +} diff --git a/pkg/webhook/controlplane/genericmutator/mutator.go b/pkg/webhook/controlplane/genericmutator/mutator.go index ef64776d3..a81e57978 100644 --- a/pkg/webhook/controlplane/genericmutator/mutator.go +++ b/pkg/webhook/controlplane/genericmutator/mutator.go @@ -17,6 +17,7 @@ package genericmutator import ( "context" extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/coreos/go-systemd/unit" @@ -51,6 +52,10 @@ type Ensurer interface { EnsureKubeletConfiguration(context.Context, *kubeletconfigv1beta1.KubeletConfiguration) error // EnsureKubernetesGeneralConfiguration ensures that the kubernetes general configuration conforms to the provider requirements. EnsureKubernetesGeneralConfiguration(context.Context, *string) error + //ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.config file should be added to the kubelet configuration. + ShouldProvisionKubeletCloudProviderConfig() bool + //EnsureKubeletCloudProviderConfig ensures that the cloudprovider.config file content conforms to the provider requirements. + EnsureKubeletCloudProviderConfig(context.Context, *string, string) error } // NewMutator creates a new controlplane mutator. @@ -139,6 +144,13 @@ func (m *mutator) mutateOperatingSystemConfig(ctx context.Context, osc *extensio } } + // Check if cloudprovider.conf needs to be ensured + if m.ensurer.ShouldProvisionKubeletCloudProviderConfig() { + if err := m.ensureKubeletCloudProviderConfig(ctx, osc); err != nil { + return err + } + } + return nil } @@ -193,3 +205,40 @@ func (m *mutator) ensureKubernetesGeneralConfigFileContent(ctx context.Context, return nil } + +const cloudProviderConfigPath = "/var/lib/kubelet/cloudprovider.conf" + +func (m *mutator) ensureKubeletCloudProviderConfig(ctx context.Context, osc *extensionsv1alpha1.OperatingSystemConfig) error { + // Create file if it does not exist + f := controlplane.FileWithPath(osc.Spec.Files, cloudProviderConfigPath) + if f == nil { + f = &extensionsv1alpha1.File{ + Path: cloudProviderConfigPath, + } + osc.Spec.Files = append(osc.Spec.Files, *f) + } + + // Ensure permissions and content inline encoding + f = controlplane.FileWithPath(osc.Spec.Files, cloudProviderConfigPath) + f.Permissions = util.Int32Ptr(0644) + f.Content = extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: "b64", + }, + } + + // Ensure content inline data + data := f.Content.Inline.Data + err := m.ensurer.EnsureKubeletCloudProviderConfig(ctx, &data, osc.Namespace) + if err != nil { + return err + } + f.Content.Inline.Data = data + + err = m.client.Update(ctx, osc) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/webhook/controlplane/genericmutator/mutator_test.go b/pkg/webhook/controlplane/genericmutator/mutator_test.go index 1e3260b8a..f344e0f9a 100644 --- a/pkg/webhook/controlplane/genericmutator/mutator_test.go +++ b/pkg/webhook/controlplane/genericmutator/mutator_test.go @@ -52,6 +52,9 @@ const ( oldKubernetesGeneralConfigData = "# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks\nnet.ipv4.tcp_tw_reuse = 1" newKubernetesGeneralConfigData = "# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks\nnet.ipv4.tcp_tw_reuse = 1\n# Provider specific settings" + + encoding = "b64" + cloudproviderconf = "W0dsb2JhbF1cbmF1dGgtdXJsOiBodHRwczovL2NsdXN0ZXIuZXUtZGUtMjAwLmNsb3VkLnNhcDo1MDAwL3Yz" ) const ( @@ -268,7 +271,7 @@ var _ = Describe("Mutator", func() { Data: oldKubernetesGeneralConfigData, } osc = &extensionsv1alpha1.OperatingSystemConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}, Spec: extensionsv1alpha1.OperatingSystemConfigSpec{ Purpose: extensionsv1alpha1.OperatingSystemConfigPurposeReconcile, Units: []extensionsv1alpha1.Unit{ @@ -339,6 +342,13 @@ var _ = Describe("Mutator", func() { return nil }, ) + ensurer.EXPECT().ShouldProvisionKubeletCloudProviderConfig().Return(true) + ensurer.EXPECT().EnsureKubeletCloudProviderConfig(context.TODO(), gomock.Any(), osc.Namespace).DoAndReturn( + func(ctx context.Context, data *string, _ string) error { + *data = cloudproviderconf + return nil + }, + ) // Create mock UnitSerializer us := mockcontrolplane.NewMockUnitSerializer(ctrl) @@ -350,11 +360,17 @@ var _ = Describe("Mutator", func() { kcc.EXPECT().Decode(&extensionsv1alpha1.FileContentInline{Data: oldKubeletConfigData}).Return(oldKubeletConfig, nil) kcc.EXPECT().Encode(newKubeletConfig, "").Return(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData}, nil) + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Update(context.TODO(), gomock.Any()).Times(1) + // Create mutator mutator := NewMutator(ensurer, us, kcc, logger) + err := mutator.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) // Call Mutate method and check the result - err := mutator.Mutate(context.TODO(), osc) + err = mutator.Mutate(context.TODO(), osc) Expect(err).To(Not(HaveOccurred())) checkOperatingSystemConfig(osc) }) @@ -371,6 +387,11 @@ func checkOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) { g := controlplane.FileWithPath(osc.Spec.Files, "/etc/sysctl.d/99-k8s-general.conf") Expect(g).To(Not(BeNil())) Expect(g.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: newKubernetesGeneralConfigData})) + c := controlplane.FileWithPath(osc.Spec.Files, "/var/lib/kubelet/cloudprovider.conf") + Expect(c).To(Not(BeNil())) + Expect(c.Path).To(Equal(cloudProviderConfigPath)) + Expect(c.Permissions).To(Equal(util.Int32Ptr(0644))) + Expect(c.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: cloudproviderconf, Encoding: encoding})) } func clientGet(result runtime.Object) interface{} { diff --git a/pkg/webhook/controlplane/genericmutator/noopensurer.go b/pkg/webhook/controlplane/genericmutator/noopensurer.go index 1ef44a8e3..daec3a116 100644 --- a/pkg/webhook/controlplane/genericmutator/noopensurer.go +++ b/pkg/webhook/controlplane/genericmutator/noopensurer.go @@ -67,3 +67,13 @@ func (e *NoopEnsurer) EnsureKubeletConfiguration(context.Context, *kubeletconfig func (e *NoopEnsurer) EnsureKubernetesGeneralConfiguration(context.Context, *string) error { return nil } + +//ShouldProvisionKubeletCloudProviderConfig returns if the cloudprovider.conf file should be added to the kubelet configuration. +func (e *NoopEnsurer) ShouldProvisionKubeletCloudProviderConfig() bool { + return false +} + +//EnsureKubeletCloudProviderConfig ensures that the cloudprovider.conf file conforms to the provider requirements. +func (e *NoopEnsurer) EnsureKubeletCloudProviderConfig(context.Context, *string, string) error { + return nil +}