[WIP] Add controlplane webhooks for Openstack

Makefile

@@ -144,6 +144,9 @@ start-provider-openstack:
 		--config-file=./controllers/provider-openstack/example/00-componentconfig.yaml \
 		--infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION) \
 		--leader-election=$(LEADER_ELECTION) \
+		--webhook-config-mode=url \
+		--webhook-config-name=openstack-webhooks \
+		--webhook-config-host=$(HOSTNAME)
 
 .PHONY: start-provider-alicloud
 start-provider-alicloud:

controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml

@@ -15,3 +15,9 @@ data:
     machineImages:
 {{ toYaml .Values.config.machineImages | indent 4 }}
 {{- end }}
+    etcd:
+      storage:
+        className: {{ .Values.config.etcd.storage.className }}
+        capacity: {{ .Values.config.etcd.storage.capacity }}
+      backup:
+        schedule: {{ .Values.config.etcd.backup.schedule }}

controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+    name: {{ .Values.config.etcd.storage.className }}
+provisioner: kubernetes.io/cinder
+allowVolumeExpansion: true
+parameters: []

controllers/provider-openstack/charts/provider-openstack/values.yaml

@@ -6,14 +6,21 @@ image:
 resources: {}
 
 controllers:
+  controlplane:
+    concurrentSyncs: 5
   infrastructure:
     concurrentSyncs: 5
     ignoreOperationAnnotation: false
   worker:
     concurrentSyncs: 5
 
 
-disableControllers: []
+disableControllers:
+  - controlplane-controller
+disableWebhooks:
+  - controlplane
+  - controlplaneexposure
+  - controlplanebackup
 
 config:
   machineImages:
@@ -22,3 +29,9 @@ config:
     cloudProfiles:
     - name: eu-de-1
       image: coreos-2023.5.0
+  etcd:
+    storage:
+      className: gardener.cloud-fast
+      capacity: 25Gi
+    backup:
+      schedule: "0 */24 * * *"

controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go

@@ -22,13 +22,15 @@ import (
 	openstackinstall "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/openstack/install"
 	openstackcmd "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/cmd"
 	openstackcp "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/controlplane"
+	openstackcontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup"
+	openstackcontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure"
 	openstackinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure"
 	openstackworker "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker"
 	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack"
 	"github.com/gardener/gardener-extensions/pkg/controller"
 	controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd"
 	"github.com/gardener/gardener-extensions/pkg/controller/infrastructure"
-
+	webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd"
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
@@ -63,7 +65,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 			MaxConcurrentReconciles: 5,
 		}
 
-		controllerSwitches = openstackcmd.ControllerSwitchOptions()
+		controllerSwitches   = openstackcmd.ControllerSwitchOptions()
+		webhookSwitches      = openstackcmd.WebhookSwitchOptions()
+		webhookServerOptions = &webhookcmd.ServerOptions{
+			Port:             7890,
+			CertDir:          "/tmp/cert",
+			Mode:             webhookcmd.ServiceMode,
+			Name:             "webhooks",
+			Namespace:        os.Getenv("WEBHOOK_CONFIG_NAMESPACE"),
+			ServiceSelectors: "{}",
+			Host:             "localhost",
+		}
+		webhookOptions = webhookcmd.NewAddToManagerOptions("openstack-webhooks", webhookServerOptions, webhookSwitches)
 
 		aggOption = controllercmd.NewOptionAggregator(
 			restOpts,
@@ -72,6 +85,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 			controllercmd.PrefixOption("infrastructure-", &infraCtrlOptsUnprefixed),
 			controllercmd.PrefixOption("worker-", workerCtrlOpts),
 			controllerSwitches,
+			configFileOpts,
+			webhookOptions,
 		)
 	)
 
@@ -101,6 +116,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 			}
 
 			configFileOpts.Completed().ApplyMachineImages(&openstackworker.DefaultAddOptions.MachineImagesToCloudProfilesMapping)
+			configFileOpts.Completed().ApplyETCDStorage(&openstackcontrolplaneexposure.DefaultAddOptions.ETCDStorage)
+			configFileOpts.Completed().ApplyETCDBackup(&openstackcontrolplanebackup.DefaultAddOptions.ETCDBackup)
 			controlPlaneCtrlOpts.Completed().Apply(&openstackcp.Options)
 			infraCtrlOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.Controller)
 			infraReconcileOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation)
@@ -110,6 +127,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 				controllercmd.LogErrAndExit(err, "Could not add controllers to manager")
 			}
 
+			if err := webhookOptions.Completed().AddToManager(mgr); err != nil {
+				controllercmd.LogErrAndExit(err, "Could not add webhooks to manager")
+			}
+
 			if err := mgr.Start(ctx.Done()); err != nil {
 				controllercmd.LogErrAndExit(err, "Error running manager")
 			}

controllers/provider-openstack/example/00-componentconfig.yaml

@@ -7,3 +7,9 @@ machineImages:
   cloudProfiles:
   - name: eu-de-1
     image: coreos-2023.5.0
+etcd:
+  storage:
+    className: gardener.cloud-fast
+    capacity: 25Gi
+  backup:
+    schedule: "0 */24 * * *"

controllers/provider-openstack/pkg/apis/config/types.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -27,6 +28,9 @@ type ControllerConfiguration struct {
 	// MachineImages is the list of machine images that are understood by the controller. It maps
 	// logical names and versions to OpenStack-specific identifiers.
 	MachineImages []MachineImage
+
+	// ETCD is the etcd configuration.
+	ETCD ETCD
 }
 
 // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers.
@@ -46,3 +50,25 @@ type CloudProfileMapping struct {
 	// Image is the name of the image.
 	Image string
 }
+
+// ETCD is an etcd configuration.
+type ETCD struct {
+	// ETCDStorage is the etcd storage configuration.
+	Storage ETCDStorage
+	// ETCDBackup is the etcd backup configuration.
+	Backup ETCDBackup
+}
+
+// ETCDStorage is an etcd storage configuration.
+type ETCDStorage struct {
+	// ClassName is the name of the storage class used in etcd-main volume claims.
+	ClassName *string
+	// Capacity is the storage capacity used in etcd-main volume claims.
+	Capacity *resource.Quantity
+}
+
+// ETCDBackup is an etcd backup configuration.
+type ETCDBackup struct {
+	// Schedule is the etcd backup schedule.
+	Schedule *string
+}

controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go

@@ -15,6 +15,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -27,6 +28,9 @@ type ControllerConfiguration struct {
 	// MachineImages is the list of machine images that are understood by the controller. It maps
 	// logical names and versions to OpenStack-specific identifiers.
 	MachineImages []MachineImage `json:"machineImages,omitempty"`
+
+	// ETCD is the etcd configuration.
+	ETCD ETCD `json:"etcd"`
 }
 
 // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers.
@@ -46,3 +50,28 @@ type CloudProfileMapping struct {
 	// Image is the name of the image.
 	Image string `json:"image"`
 }
+
+// ETCD is an etcd configuration.
+type ETCD struct {
+	// ETCDStorage is the etcd storage configuration.
+	Storage ETCDStorage `json:"storage"`
+	// ETCDBackup is the etcd backup configuration.
+	Backup ETCDBackup `json:"backup"`
+}
+
+// ETCDStorage is an etcd storage configuration.
+type ETCDStorage struct {
+	// ClassName is the name of the storage class used in etcd-main volume claims.
+	// +optional
+	ClassName *string `json:"className,omitempty"`
+	// Capacity is the storage capacity used in etcd-main volume claims.
+	// +optional
+	Capacity *resource.Quantity `json:"capacity,omitempty"`
+}
+
+// ETCDBackup is an etcd backup configuration.
+type ETCDBackup struct {
+	// Schedule is the etcd backup schedule.
+	// +optional
+	Schedule *string `json:"schedule,omitempty"`
+}

controllers/provider-openstack/pkg/cmd/config.go

@@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) {
 	*machineImages = c.Config.MachineImages
 }
 
+// ApplyETCDStorage sets the given etcd storage configuration to that of this Config.
+func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) {
+	*etcdStorage = c.Config.ETCD.Storage
+}
+
+// ApplyETCDBackup sets the given etcd backup configuration to that of this Config.
+func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) {
+	*etcdBackup = c.Config.ETCD.Backup
+}
+
 // Options initializes empty config.ControllerConfiguration, applies the set values and returns it.
 func (c *Config) Options() config.ControllerConfiguration {
 	var cfg config.ControllerConfiguration

controllers/provider-openstack/pkg/cmd/options.go

@@ -18,10 +18,16 @@ import (
 	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/controlplane"
 	infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure"
 	workercontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker"
+	controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplane"
+	controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup"
+	controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure"
 	controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd"
 	extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane"
 	extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure"
 	extensionsworkercontroller "github.com/gardener/gardener-extensions/pkg/controller/worker"
+
+	webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd"
+	extensioncontrolplanewebhook "github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
 )
 
 // ControllerSwitchOptions are the controllercmd.SwitchOptions for the provider controllers.
@@ -32,3 +38,12 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions {
 		controllercmd.Switch(extensionsworkercontroller.ControllerName, workercontroller.AddToManager),
 	)
 }
+
+// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks.
+func WebhookSwitchOptions() *webhookcmd.SwitchOptions {
+	return webhookcmd.NewSwitchOptions(
+		webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager),
+		webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager),
+		webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager),
+	)
+}

controllers/provider-openstack/pkg/openstack/types.go

@@ -19,18 +19,18 @@ import "path/filepath"
 const (
 	// Name is the name of the OpenStack provider.
 	Name = "provider-openstack"
+	// StorageProviderName is the name of the Openstack storage provider.
+	StorageProviderName = "Swift"
 
 	// MachineControllerManagerImageName is the name of the MachineControllerManager image.
 	MachineControllerManagerImageName = "machine-controller-manager"
 	// HyperkubeImageName is the name of the hyperkube image.
 	HyperkubeImageName = "hyperkube"
+	// ETCDBackupRestoreImageName is the name of the etcd backup and restore image.
+	ETCDBackupRestoreImageName = "etcd-backup-restore"
 
 	// AuthURL is a constant for the key in a cloud provider secret that holds the OpenStack auth url.
 	AuthURL = "authURL"
-
-	// CloudProviderConfigName is the name of the configmap containing the cloud provider config.
-	CloudProviderConfigName = "cloud-provider-config"
-
 	// DomainName is a constant for the key in a cloud provider secret that holds the OpenStack domain name.
 	DomainName = "domainName"
 	// TenantName is a constant for the key in a cloud provider secret that holds the OpenStack tenant name.
@@ -40,8 +40,18 @@ const (
 	// Password is a constant for the key in a cloud provider secret and backup secret that holds the OpenStack password.
 	Password = "password"
 
+	// BucketName is a constant for the key in a backup secret that holds the bucket name.
+	// The bucket name is written to the backup secret by Gardener as a temporary solution.
+	// TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md)
+	BucketName = "bucketName"
+
+	// CloudProviderConfigName is the name of the configmap containing the cloud provider config.
+	CloudProviderConfigName = "cloud-provider-config"
 	// MachineControllerManagerName is a constant for the name of the machine-controller-manager.
 	MachineControllerManagerName = "machine-controller-manager"
+	// BackupSecretName defines the name of the secret containing the credentials which are required to
+	// authenticate against the respective cloud provider (required to store the backups of Shoot clusters).
+	BackupSecretName = "etcd-backup"
 )
 
 var (

controllers/provider-openstack/pkg/webhook/controlplane/app.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack"
+	extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook"
+	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane"
+	"github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator"
+
+	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+var logger = log.Log.WithName("openstack-controlplane-webhook")
+
+// AddToManager creates a webhook and adds it to the manager.
+func AddToManager(mgr manager.Manager) (webhook.Webhook, error) {
+	logger.Info("Adding webhook to manager")
+	return controlplane.Add(mgr, controlplane.AddArgs{
+		Kind:     extensionswebhook.ShootKind,
+		Provider: openstack.Type,
+		Types:    []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}},
+		Mutator: genericmutator.NewMutator(NewEnsurer(logger), controlplane.NewUnitSerializer(),
+			controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger),
+	})
+}