diff --git a/Makefile b/Makefile index e5a6df97b..15fc41f6e 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,9 @@ start-provider-openstack: --config-file=./controllers/provider-openstack/example/00-componentconfig.yaml \ --infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION) \ --leader-election=$(LEADER_ELECTION) \ + --webhook-config-mode=url \ + --webhook-config-name=openstack-webhooks \ + --webhook-config-host=$(HOSTNAME) .PHONY: start-provider-alicloud start-provider-alicloud: diff --git a/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml b/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml index 6cadac998..ec1ef5527 100644 --- a/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml +++ b/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml @@ -15,3 +15,9 @@ data: machineImages: {{ toYaml .Values.config.machineImages | indent 4 }} {{- end }} + etcd: + storage: + className: {{ .Values.config.etcd.storage.className }} + capacity: {{ .Values.config.etcd.storage.capacity }} + backup: + schedule: {{ .Values.config.etcd.backup.schedule }} \ No newline at end of file diff --git a/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml b/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml new file mode 100644 index 000000000..a5213ed20 --- /dev/null +++ b/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.config.etcd.storage.className }} +provisioner: kubernetes.io/cinder +allowVolumeExpansion: true +parameters: [] \ No newline at end of file diff --git a/controllers/provider-openstack/charts/provider-openstack/values.yaml b/controllers/provider-openstack/charts/provider-openstack/values.yaml index bec154928..257096d33 100644 --- a/controllers/provider-openstack/charts/provider-openstack/values.yaml +++ b/controllers/provider-openstack/charts/provider-openstack/values.yaml @@ -6,6 +6,8 @@ image: resources: {} controllers: + controlplane: + concurrentSyncs: 5 infrastructure: concurrentSyncs: 5 ignoreOperationAnnotation: false @@ -13,7 +15,12 @@ controllers: concurrentSyncs: 5 -disableControllers: [] +disableControllers: + - controlplane-controller +disableWebhooks: + - controlplane + - controlplaneexposure + - controlplanebackup config: machineImages: @@ -22,3 +29,9 @@ config: cloudProfiles: - name: eu-de-1 image: coreos-2023.5.0 + etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" \ No newline at end of file diff --git a/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go b/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go index ad2c90592..a9aeed249 100644 --- a/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go +++ b/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go @@ -22,13 +22,15 @@ import ( openstackinstall "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/openstack/install" openstackcmd "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/cmd" openstackcp "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/controlplane" + openstackcontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup" + openstackcontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure" openstackinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure" openstackworker "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker" "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" - + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -63,7 +65,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { MaxConcurrentReconciles: 5, } - controllerSwitches = openstackcmd.ControllerSwitchOptions() + controllerSwitches = openstackcmd.ControllerSwitchOptions() + webhookSwitches = openstackcmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("openstack-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, @@ -72,6 +85,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.PrefixOption("infrastructure-", &infraCtrlOptsUnprefixed), controllercmd.PrefixOption("worker-", workerCtrlOpts), controllerSwitches, + configFileOpts, + webhookOptions, ) ) @@ -101,6 +116,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts.Completed().ApplyMachineImages(&openstackworker.DefaultAddOptions.MachineImagesToCloudProfilesMapping) + configFileOpts.Completed().ApplyETCDStorage(&openstackcontrolplaneexposure.DefaultAddOptions.ETCDStorage) + configFileOpts.Completed().ApplyETCDBackup(&openstackcontrolplanebackup.DefaultAddOptions.ETCDBackup) controlPlaneCtrlOpts.Completed().Apply(&openstackcp.Options) infraCtrlOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.Controller) infraReconcileOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation) @@ -110,6 +127,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { + controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") + } + if err := mgr.Start(ctx.Done()); err != nil { controllercmd.LogErrAndExit(err, "Error running manager") } diff --git a/controllers/provider-openstack/example/00-componentconfig.yaml b/controllers/provider-openstack/example/00-componentconfig.yaml index 51f4fd064..af91a023a 100644 --- a/controllers/provider-openstack/example/00-componentconfig.yaml +++ b/controllers/provider-openstack/example/00-componentconfig.yaml @@ -7,3 +7,9 @@ machineImages: cloudProfiles: - name: eu-de-1 image: coreos-2023.5.0 +etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" \ No newline at end of file diff --git a/controllers/provider-openstack/pkg/apis/config/types.go b/controllers/provider-openstack/pkg/apis/config/types.go index a956a5d95..a6392473d 100644 --- a/controllers/provider-openstack/pkg/apis/config/types.go +++ b/controllers/provider-openstack/pkg/apis/config/types.go @@ -15,6 +15,7 @@ package config import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,9 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to OpenStack-specific identifiers. MachineImages []MachineImage + + // ETCD is the etcd configuration. + ETCD ETCD } // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers. @@ -46,3 +50,25 @@ type CloudProfileMapping struct { // Image is the name of the image. Image string } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + ClassName *string + // Capacity is the storage capacity used in etcd-main volume claims. + Capacity *resource.Quantity +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + Schedule *string +} diff --git a/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go b/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go index 2697e233a..69ac79213 100644 --- a/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go +++ b/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,9 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to OpenStack-specific identifiers. MachineImages []MachineImage `json:"machineImages,omitempty"` + + // ETCD is the etcd configuration. + ETCD ETCD `json:"etcd"` } // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers. @@ -46,3 +50,28 @@ type CloudProfileMapping struct { // Image is the name of the image. Image string `json:"image"` } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage `json:"storage"` + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup `json:"backup"` +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + // +optional + ClassName *string `json:"className,omitempty"` + // Capacity is the storage capacity used in etcd-main volume claims. + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty"` +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + // +optional + Schedule *string `json:"schedule,omitempty"` +} diff --git a/controllers/provider-openstack/pkg/cmd/config.go b/controllers/provider-openstack/pkg/cmd/config.go index ba8043da7..bcfb659f7 100644 --- a/controllers/provider-openstack/pkg/cmd/config.go +++ b/controllers/provider-openstack/pkg/cmd/config.go @@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) { *machineImages = c.Config.MachineImages } +// ApplyETCDStorage sets the given etcd storage configuration to that of this Config. +func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { + *etcdStorage = c.Config.ETCD.Storage +} + +// ApplyETCDBackup sets the given etcd backup configuration to that of this Config. +func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) { + *etcdBackup = c.Config.ETCD.Backup +} + // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/controllers/provider-openstack/pkg/cmd/options.go b/controllers/provider-openstack/pkg/cmd/options.go index 94997f1ed..9d08eb47b 100644 --- a/controllers/provider-openstack/pkg/cmd/options.go +++ b/controllers/provider-openstack/pkg/cmd/options.go @@ -18,10 +18,16 @@ import ( "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/controlplane" infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker" + controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplane" + controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup" + controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane" extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" extensionsworkercontroller "github.com/gardener/gardener-extensions/pkg/controller/worker" + + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" + extensioncontrolplanewebhook "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" ) // ControllerSwitchOptions are the controllercmd.SwitchOptions for the provider controllers. @@ -32,3 +38,12 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { controllercmd.Switch(extensionsworkercontroller.ControllerName, workercontroller.AddToManager), ) } + +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions( + webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager), + ) +} diff --git a/controllers/provider-openstack/pkg/openstack/types.go b/controllers/provider-openstack/pkg/openstack/types.go index 0d052ac5a..6d388cfd3 100644 --- a/controllers/provider-openstack/pkg/openstack/types.go +++ b/controllers/provider-openstack/pkg/openstack/types.go @@ -19,18 +19,18 @@ import "path/filepath" const ( // Name is the name of the OpenStack provider. Name = "provider-openstack" + // StorageProviderName is the name of the Openstack storage provider. + StorageProviderName = "Swift" // MachineControllerManagerImageName is the name of the MachineControllerManager image. MachineControllerManagerImageName = "machine-controller-manager" // HyperkubeImageName is the name of the hyperkube image. HyperkubeImageName = "hyperkube" + // ETCDBackupRestoreImageName is the name of the etcd backup and restore image. + ETCDBackupRestoreImageName = "etcd-backup-restore" // AuthURL is a constant for the key in a cloud provider secret that holds the OpenStack auth url. AuthURL = "authURL" - - // CloudProviderConfigName is the name of the configmap containing the cloud provider config. - CloudProviderConfigName = "cloud-provider-config" - // DomainName is a constant for the key in a cloud provider secret that holds the OpenStack domain name. DomainName = "domainName" // TenantName is a constant for the key in a cloud provider secret that holds the OpenStack tenant name. @@ -40,8 +40,18 @@ const ( // Password is a constant for the key in a cloud provider secret and backup secret that holds the OpenStack password. Password = "password" + // BucketName is a constant for the key in a backup secret that holds the bucket name. + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + BucketName = "bucketName" + + // CloudProviderConfigName is the name of the configmap containing the cloud provider config. + CloudProviderConfigName = "cloud-provider-config" // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" + // BackupSecretName defines the name of the secret containing the credentials which are required to + // authenticate against the respective cloud provider (required to store the backups of Shoot clusters). + BackupSecretName = "etcd-backup" ) var ( diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/app.go b/controllers/provider-openstack/pkg/webhook/controlplane/app.go new file mode 100644 index 000000000..5fcb09fa1 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/app.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = log.Log.WithName("openstack-controlplane-webhook") + +// AddToManager creates a webhook and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.ShootKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(logger), controlplane.NewUnitSerializer(), + controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), + }) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go new file mode 100644 index 000000000..faf92b5df --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go @@ -0,0 +1,145 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/coreos/go-systemd/unit" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// NewEnsurer creates a new controlplane ensurer. +func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + logger: logger.WithName("openstack-controlplane-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + logger logr.Logger +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-apiserver"); c != nil { + ensureKubeAPIServerCommandLineArgs(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +// EnsureKubeControllerManagerDeployment ensures that the kube-controller-manager deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeControllerManagerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-controller-manager"); c != nil { + ensureKubeControllerManagerCommandLineArgs(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "openstack") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = controlplane.EnsureNoStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") +} + +func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-cloud-volume-plugin=", "openstack") +} + +var ( + cloudProviderConfigVolumeMount = corev1.VolumeMount{ + Name: openstack.CloudProviderConfigName, + MountPath: "/etc/kubernetes/cloudprovider", + } + cloudProviderSecretVolumeMount = corev1.VolumeMount{ + Name: common.CloudProviderSecretName, + MountPath: "/srv/cloudprovider", + } + + cloudProviderConfigVolume = corev1.Volume{ + Name: openstack.CloudProviderConfigName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.CloudProviderConfigName}, + }, + }, + } + cloudProviderSecretVolume = corev1.Volume{ + Name: common.CloudProviderSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + SecretName: common.CloudProviderSecretName, + }, + }, + } +) + +func ensureVolumeMounts(c *corev1.Container) { + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderConfigVolumeMount) + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderSecretVolumeMount) +} + +func ensureVolumes(ps *corev1.PodSpec) { + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderConfigVolume) + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderSecretVolume) +} + +// EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. +func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, opts []*unit.UnitOption) ([]*unit.UnitOption, error) { + if opt := controlplane.UnitOptionWithSectionAndName(opts, "Service", "ExecStart"); opt != nil { + command := controlplane.DeserializeCommandLine(opt.Value) + command = ensureKubeletCommandLineArgs(command) + opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n ") + } + return opts, nil +} + +func ensureKubeletCommandLineArgs(command []string) []string { + command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "openstack") + return command +} + +// EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. +func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { + // Make sure CSI-related feature gates are not enabled + // TODO Leaving these enabled shouldn't do any harm, perhaps remove this code when properly tested? + delete(kubeletConfig.FeatureGates, "VolumeSnapshotDataSource") + delete(kubeletConfig.FeatureGates, "CSINodeInfo") + delete(kubeletConfig.FeatureGates, "CSIDriverRegistry") + return nil +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go new file mode 100644 index 000000000..80472d560 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go @@ -0,0 +1,286 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test" + + "github.com/coreos/go-systemd/unit" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + ctrl *gomock.Controller + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--enable-admission-plugins=Priority,NamespaceLifecycle", + "--disable-admission-plugins=PersistentVolumeLabel", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: openstack.CloudProviderConfigName, MountPath: "?"}, + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: openstack.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureKubeControllerManagerDeployment", func() { + It("should add missing elements to kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--external-cloud-volume-plugin=?", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: openstack.CloudProviderConfigName, MountPath: "?"}, + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: openstack.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + }) + + Describe("#EnsureKubeletServiceUnitOptions", func() { + It("should modify existing elements of kubelet.service unit options", func() { + var ( + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --cloud-provider=openstack`, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletServiceUnitOptions method and check the result + opts, err := ensurer.EnsureKubeletServiceUnitOptions(context.TODO(), oldUnitOptions) + Expect(err).To(Not(HaveOccurred())) + Expect(opts).To(Equal(newUnitOptions)) + }) + }) + + Describe("#EnsureKubeletConfiguration", func() { + It("should modify existing elements of kubelet configuration", func() { + var ( + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletConfiguration method and check the result + kubeletConfig := *oldKubeletConfig + err := ensurer.EnsureKubeletConfiguration(context.TODO(), &kubeletConfig) + Expect(err).To(Not(HaveOccurred())) + Expect(&kubeletConfig).To(Equal(newKubeletConfig)) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=openstack")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ",")) + Expect(c.Command).To(Not(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ","))) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} + +func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { + // Check that the kube-controller-manager container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=external")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(ContainElement("--external-cloud-volume-plugin=openstack")) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go new file mode 100644 index 000000000..cb3f51dc7 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/imagevector" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the Openstack backup webhook to the manager. +type AddOptions struct { + // ETCDBackup is the etcd backup configuration. + ETCDBackup config.ETCDBackup +} + +var logger = log.Log.WithName("openstack-controlplanebackup-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.BackupKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDBackup, imagevector.ImageVector(), logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go new file mode 100644 index 000000000..141004afd --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go @@ -0,0 +1,153 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdBackup *config.ETCDBackup, imageVector imagevector.ImageVector, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdBackup: etcdBackup, + imageVector: imageVector, + logger: logger.WithName("openstack-controlplanebackup-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdBackup *config.ETCDBackup + imageVector imagevector.ImageVector + logger logr.Logger +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + if err := e.ensureContainers(&ss.Spec.Template.Spec, ss.Name, cluster); err != nil { + return err + } + return nil +} + +func (e *ensurer) ensureContainers(ps *corev1.PodSpec, name string, cluster *extensionscontroller.Cluster) error { + c, err := e.getBackupRestoreContainer(name, cluster) + if err != nil { + return err + } + ps.Containers = controlplane.EnsureContainerWithName(ps.Containers, *c) + return nil +} + +func (e *ensurer) getBackupRestoreContainer(name string, cluster *extensionscontroller.Cluster) (*corev1.Container, error) { + // Find etcd-backup-restore image + // TODO Get seed version from clientset when it's possible to inject it + image, err := e.imageVector.FindImage(openstack.ETCDBackupRestoreImageName, "", cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, errors.Wrapf(err, "could not find image %s", openstack.ETCDBackupRestoreImageName) + } + + const ( + defaultSchedule = "0 */24 * * *" + ) + + // Determine provider, container env variables, and volume mounts + // They are only specified for the etcd-main stateful set (backup is enabled) + var ( + provider string + env []corev1.EnvVar + ) + if name == common.EtcdMainStatefulSetName { + provider = openstack.StorageProviderName + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: openstack.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + }, + }, + }, + { + Name: "OS_AUTH_URL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.AuthURL, + }, + }, + }, + { + Name: "OS_DOMAIN_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.DomainName, + }, + }, + }, + { + Name: "OS_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.UserName, + }, + }, + }, + { + Name: "OS_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.Password, + }, + }, + }, + { + Name: "OS_TENANT_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.TenantName, + }, + }, + }, + } + } + + // Determine schedule + var schedule = defaultSchedule + if e.etcdBackup.Schedule != nil { + schedule = defaultSchedule + } + + return controlplane.GetBackupRestoreContainer(name, schedule, provider, image.String(), nil, env, nil), nil +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go new file mode 100644 index 000000000..710928610 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Backup Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + Describe("#EnsureETCDStatefulSet", func() { + var ( + etcdBackup = &config.ETCDBackup{ + Schedule: util.StringPtr("0 */24 * * *"), + } + + imageVector = imagevector.ImageVector{ + { + Name: openstack.ETCDBackupRestoreImageName, + Repository: "test-repository", + Tag: "test-tag", + }, + } + + cluster = &extensionscontroller.Cluster{ + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + ) + + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + var ( + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: openstack.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + }, + }, + }, + { + Name: "OS_AUTH_URL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.AuthURL, + }, + }, + }, + { + Name: "OS_DOMAIN_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.DomainName, + }, + }, + }, + { + Name: "OS_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.UserName, + }, + }, + }, + { + Name: "OS_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.Password, + }, + }, + }, + { + Name: "OS_TENANT_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.TenantName, + }, + }, + }, + } + ) + + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdMainStatefulSetName, "0 */24 * * *", openstack.StorageProviderName, + "test-repository:test-tag", nil, env, nil))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdEventsStatefulSetName, "0 */24 * * *", "", + "test-repository:test-tag", nil, nil, nil))) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go new file mode 100644 index 000000000..cb7559241 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the Openstack exposure webhook to the manager. +type AddOptions struct { + // ETCDStorage is the etcd storage configuration. + ETCDStorage config.ETCDStorage +} + +var logger = log.Log.WithName("openstack-controlplaneexposure-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.SeedKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &corev1.Service{}, &appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDStorage, logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go new file mode 100644 index 000000000..8f4704493 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go @@ -0,0 +1,85 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdStorage *config.ETCDStorage, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdStorage: etcdStorage, + logger: logger.WithName("openstack-controlplaneexposure-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdStorage *config.ETCDStorage + client client.Client + logger logr.Logger +} + +// InjectClient injects the given client into the ensurer. +func (m *ensurer) InjectClient(client client.Client) error { + m.client = client + return nil +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + // Get load balancer address of the kube-apiserver service + address, err := controlplane.GetLoadBalancerIngress(ctx, e.client, dep.Namespace, common.KubeAPIServerDeploymentName) + if err != nil { + return errors.Wrap(err, "could not get kube-apiserver service load balancer address") + } + + if c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver"); c != nil { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--advertise-address=", address) + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-hostname=", address) + } + return nil +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + e.ensureVolumeClaimTemplates(&ss.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureVolumeClaimTemplates(spec *appsv1.StatefulSetSpec, name string) { + t := e.getVolumeClaimTemplate(name) + spec.VolumeClaimTemplates = controlplane.EnsurePVCWithName(spec.VolumeClaimTemplates, *t) +} + +func (e *ensurer) getVolumeClaimTemplate(name string) *corev1.PersistentVolumeClaim { + var etcdStorage config.ETCDStorage + if name == common.EtcdMainStatefulSetName { + etcdStorage = *e.etcdStorage + } + return controlplane.GetETCDVolumeClaimTemplate(name, etcdStorage.ClassName, etcdStorage.Capacity) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go new file mode 100644 index 000000000..40fbe9689 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Exposure Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + etcdStorage = &config.ETCDStorage{ + ClassName: util.StringPtr("gardener.cloud-fast"), + Capacity: util.QuantityPtr(resource.MustParse("25Gi")), + } + + ctrl *gomock.Controller + + svcKey = client.ObjectKey{Namespace: namespace, Name: common.KubeAPIServerDeploymentName} + svc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "1.2.3.4"}, + }, + }, + }, + } + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{"--advertise-address=?", "--external-hostname=?"}, + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureETCDStatefulSet", func() { + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-main"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-events"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--advertise-address=1.2.3.4")) + Expect(c.Command).To(ContainElement("--external-hostname=1.2.3.4")) +} + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-main") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdMainStatefulSetName, util.StringPtr("gardener.cloud-fast"), + util.QuantityPtr(resource.MustParse("25Gi"))))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-events") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdEventsStatefulSetName, nil, nil))) +} + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *corev1.Service: + *obj.(*corev1.Service) = *result.(*corev1.Service) + } + return nil + } +}