From 23b275512d76cfb43a01658b3bdf45602eeba8a6 Mon Sep 17 00:00:00 2001 From: Svetlina Shopova Date: Wed, 5 Jun 2019 17:26:21 +0300 Subject: [PATCH] Add controlplane webhooks for Openstack --- Makefile | 3 + .../provider-openstack/charts/images.yaml | 4 + .../templates/configmap.yaml | 6 + .../templates/deployment.yaml | 10 + .../templates/storageclass.yaml | 7 + .../charts/provider-openstack/values.yaml | 15 +- .../app/app.go | 30 +- .../example/00-componentconfig.yaml | 6 + .../example/30-controlplane.yaml | 8 + .../example/30-etcd-backup-secret.yaml | 13 + .../example/controller-registration.yaml | 2 +- .../pkg/apis/config/types.go | 26 ++ .../pkg/apis/config/v1alpha1/types.go | 29 ++ .../v1alpha1/zz_generated.conversion.go | 109 +++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 66 ++++ .../pkg/apis/config/zz_generated.deepcopy.go | 66 ++++ .../provider-openstack/pkg/cmd/config.go | 10 + .../provider-openstack/pkg/cmd/options.go | 15 + .../provider-openstack/pkg/openstack/types.go | 18 +- .../pkg/webhook/controlplane/app.go | 43 +++ .../pkg/webhook/controlplane/ensurer.go | 145 +++++++++ .../pkg/webhook/controlplane/ensurer_test.go | 286 ++++++++++++++++++ .../pkg/webhook/controlplanebackup/add.go | 59 ++++ .../pkg/webhook/controlplanebackup/ensurer.go | 153 ++++++++++ .../controlplanebackup/ensurer_test.go | 225 ++++++++++++++ .../pkg/webhook/controlplaneexposure/add.go | 59 ++++ .../webhook/controlplaneexposure/ensurer.go | 85 ++++++ .../controlplaneexposure/ensurer_test.go | 271 +++++++++++++++++ 28 files changed, 1756 insertions(+), 13 deletions(-) create mode 100644 controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml create mode 100644 controllers/provider-openstack/example/30-etcd-backup-secret.yaml create mode 100644 controllers/provider-openstack/pkg/webhook/controlplane/app.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go create mode 100644 controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go diff --git a/Makefile b/Makefile index e5a6df97b..15fc41f6e 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,9 @@ start-provider-openstack: --config-file=./controllers/provider-openstack/example/00-componentconfig.yaml \ --infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION) \ --leader-election=$(LEADER_ELECTION) \ + --webhook-config-mode=url \ + --webhook-config-name=openstack-webhooks \ + --webhook-config-host=$(HOSTNAME) .PHONY: start-provider-alicloud start-provider-alicloud: diff --git a/controllers/provider-openstack/charts/images.yaml b/controllers/provider-openstack/charts/images.yaml index a53859762..e4503abc7 100644 --- a/controllers/provider-openstack/charts/images.yaml +++ b/controllers/provider-openstack/charts/images.yaml @@ -10,3 +10,7 @@ images: sourceRepository: github.com/gardener/machine-controller-manager repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager tag: "0.18.0" +- name: etcd-backup-restore + sourceRepository: github.com/gardener/etcd-backup-restore + repository: eu.gcr.io/gardener-project/gardener/etcdbrctl + tag: "0.6.3" \ No newline at end of file diff --git a/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml b/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml index 6cadac998..ec1ef5527 100644 --- a/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml +++ b/controllers/provider-openstack/charts/provider-openstack/templates/configmap.yaml @@ -15,3 +15,9 @@ data: machineImages: {{ toYaml .Values.config.machineImages | indent 4 }} {{- end }} + etcd: + storage: + className: {{ .Values.config.etcd.storage.className }} + capacity: {{ .Values.config.etcd.storage.capacity }} + backup: + schedule: {{ .Values.config.etcd.backup.schedule }} \ No newline at end of file diff --git a/controllers/provider-openstack/charts/provider-openstack/templates/deployment.yaml b/controllers/provider-openstack/charts/provider-openstack/templates/deployment.yaml index 06421a5d8..9cf173edb 100644 --- a/controllers/provider-openstack/charts/provider-openstack/templates/deployment.yaml +++ b/controllers/provider-openstack/charts/provider-openstack/templates/deployment.yaml @@ -30,10 +30,16 @@ spec: - /gardener-extension-hyper - provider-openstack-controller-manager - --config-file=/etc/gardener-extension-provider-openstack/config/config.yaml + - --controlplane-max-concurrent-reconciles={{ .Values.controllers.controlplane.concurrentSyncs }} - --infrastructure-max-concurrent-reconciles={{ .Values.controllers.infrastructure.concurrentSyncs }} - --infrastructure-ignore-operation-annotation={{ .Values.controllers.infrastructure.ignoreOperationAnnotation }} - --worker-max-concurrent-reconciles={{ .Values.controllers.worker.concurrentSyncs }} + - --webhook-config-mode=service + - --webhook-config-name=openstack-webhooks + - --webhook-config-namespace={{ .Release.Namespace }} + - --webhook-config-service-selectors={"app.kubernetes.io/name":"gardener-extension-provider-openstack","app.kubernetes.io/instance":"{{ .Release.Name }}"} - --disable-controllers={{ .Values.disableControllers | join "," }} + - --disable-webhooks={{ .Values.disableWebhooks | join "," }} env: - name: LEADER_ELECTION_NAMESPACE valueFrom: @@ -44,6 +50,10 @@ spec: volumeMounts: - name: config mountPath: /etc/gardener-extension-provider-openstack/config + ports: + - name: webhook-server + containerPort: 7890 + protocol: TCP volumes: - name: config configMap: diff --git a/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml b/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml new file mode 100644 index 000000000..a5213ed20 --- /dev/null +++ b/controllers/provider-openstack/charts/provider-openstack/templates/storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.config.etcd.storage.className }} +provisioner: kubernetes.io/cinder +allowVolumeExpansion: true +parameters: [] \ No newline at end of file diff --git a/controllers/provider-openstack/charts/provider-openstack/values.yaml b/controllers/provider-openstack/charts/provider-openstack/values.yaml index bec154928..257096d33 100644 --- a/controllers/provider-openstack/charts/provider-openstack/values.yaml +++ b/controllers/provider-openstack/charts/provider-openstack/values.yaml @@ -6,6 +6,8 @@ image: resources: {} controllers: + controlplane: + concurrentSyncs: 5 infrastructure: concurrentSyncs: 5 ignoreOperationAnnotation: false @@ -13,7 +15,12 @@ controllers: concurrentSyncs: 5 -disableControllers: [] +disableControllers: + - controlplane-controller +disableWebhooks: + - controlplane + - controlplaneexposure + - controlplanebackup config: machineImages: @@ -22,3 +29,9 @@ config: cloudProfiles: - name: eu-de-1 image: coreos-2023.5.0 + etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" \ No newline at end of file diff --git a/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go b/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go index ad2c90592..abc609dcf 100644 --- a/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go +++ b/controllers/provider-openstack/cmd/gardener-extension-provider-openstack/app/app.go @@ -25,10 +25,12 @@ import ( openstackinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure" openstackworker "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker" "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + openstackcontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup" + openstackcontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" - + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -63,7 +65,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { MaxConcurrentReconciles: 5, } - controllerSwitches = openstackcmd.ControllerSwitchOptions() + controllerSwitches = openstackcmd.ControllerSwitchOptions() + webhookSwitches = openstackcmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("openstack-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, @@ -72,6 +85,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.PrefixOption("infrastructure-", &infraCtrlOptsUnprefixed), controllercmd.PrefixOption("worker-", workerCtrlOpts), controllerSwitches, + configFileOpts, + webhookOptions, ) ) @@ -83,10 +98,6 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Error completing options") } - if err := configFileOpts.Complete(); err != nil { - controllercmd.LogErrAndExit(err, "Error completing config options") - } - mgr, err := manager.New(restOpts.Completed().Config, mgrOpts.Completed().Options()) if err != nil { controllercmd.LogErrAndExit(err, "Could not instantiate manager") @@ -101,6 +112,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts.Completed().ApplyMachineImages(&openstackworker.DefaultAddOptions.MachineImagesToCloudProfilesMapping) + configFileOpts.Completed().ApplyETCDStorage(&openstackcontrolplaneexposure.DefaultAddOptions.ETCDStorage) + configFileOpts.Completed().ApplyETCDBackup(&openstackcontrolplanebackup.DefaultAddOptions.ETCDBackup) controlPlaneCtrlOpts.Completed().Apply(&openstackcp.Options) infraCtrlOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.Controller) infraReconcileOpts.Completed().Apply(&openstackinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation) @@ -110,6 +123,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { + controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") + } + if err := mgr.Start(ctx.Done()); err != nil { controllercmd.LogErrAndExit(err, "Error running manager") } @@ -117,7 +134,6 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } aggOption.AddFlags(cmd.Flags()) - configFileOpts.AddFlags(cmd.Flags()) return cmd } diff --git a/controllers/provider-openstack/example/00-componentconfig.yaml b/controllers/provider-openstack/example/00-componentconfig.yaml index 51f4fd064..af91a023a 100644 --- a/controllers/provider-openstack/example/00-componentconfig.yaml +++ b/controllers/provider-openstack/example/00-componentconfig.yaml @@ -7,3 +7,9 @@ machineImages: cloudProfiles: - name: eu-de-1 image: coreos-2023.5.0 +etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" \ No newline at end of file diff --git a/controllers/provider-openstack/example/30-controlplane.yaml b/controllers/provider-openstack/example/30-controlplane.yaml index 7289bd5b3..ca18b6f74 100644 --- a/controllers/provider-openstack/example/30-controlplane.yaml +++ b/controllers/provider-openstack/example/30-controlplane.yaml @@ -1,3 +1,11 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: shoot--foo--bar # shoot--dev--local + labels: +# backup.gardener.cloud/provider: openstack +# seed.gardener.cloud/provider: openstack + shoot.gardener.cloud/provider: openstack --- apiVersion: v1 kind: Secret diff --git a/controllers/provider-openstack/example/30-etcd-backup-secret.yaml b/controllers/provider-openstack/example/30-etcd-backup-secret.yaml new file mode 100644 index 000000000..4392f3b92 --- /dev/null +++ b/controllers/provider-openstack/example/30-etcd-backup-secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-backup + namespace: shoot--foo--bar +type: Opaque +data: + bucketName: YnVja2V0MTIz + authURL: dGVzdFVSTA== + domainName: ZG9tYWluLW9wZW4tc3RhY2s= + tenantName: dGVuYW50LW9wZW4tc3RhY2s= + username: YWRtaW4= + password: YWRtaW4= \ No newline at end of file diff --git a/controllers/provider-openstack/example/controller-registration.yaml b/controllers/provider-openstack/example/controller-registration.yaml index 2ac4498c2..0101a44a8 100644 --- a/controllers/provider-openstack/example/controller-registration.yaml +++ b/controllers/provider-openstack/example/controller-registration.yaml @@ -14,7 +14,7 @@ spec: deployment: type: helm providerConfig: - chart: H4sIAAAAAAAAA+0aa2/bOLKf/SsGxn1oD5HkR5zs+bAf3DS7a1zqGHF2i+JwWNASbbORRB1J2fWl/e87JCVZsp3k0noTdKuBAUkUOS8O5yUngi9ZQIXDExpLRfwb78WhoYVw2uuZK8L21dy3u8ftTq9zcqLH293uaecF9A7OyR5IUWgB8EJwru6b99D7bxSS3f0/WxCh3DWJwgPReGj/O9321v73uu2TF9A6EP174Tvff5Kw36iQjMd9WLYbJEmKx5Z76racgC4bAZW+YIkywwP4hYYR+NpKYMYFqAWFn4kIaEwFXKIZTbQZwTizLKAfFZoWrm3EJKJ92DW5xnKX5nMr5juBPec/4L475wek8cD577S6p1vn//j4tFuf/6cAz4MznqwFmy8UvPRfQafV/gdMBmOYnAMebhKbBzKbsZARRcHnUULitQuDMASzTIKgkoolDVy4XjAJOJUCXkPmo03RANJY+wHtJwYJ8fEy4TO1IoLChZ1yBEsXOugpfJooIBJirnAdxyVixSRii83yi+HZ+QgZ0xQanoe/HMMeIgXuzKNBx23BSz2hmb1qvvqnRrHmKURkrYlCisRUIUTGEFLXYqMCYp/CiqmF5cZicTWO9xkOPlUEpxNckODTrDwRiMqYNrBQKul73mq1conh2OVi7mVKk14mq4NcZ6t+jUMqtbb/mzKBEk/XgP4aF5Ap8hqSldmwuaD4TnHN9UowxeL5EchM4RpNwKQSbJqqitJyHlH08gRUG5pAczCB4aQJrweT4eRII3k3vP7l8tdreDe4uhqMrofnE7i8grPL0Zvh9fByhE8/wWD0Hv41HL05Asr0TqI6E6ElQDaZVidajMY1obTCQh5UZEJ9NmM+ihbPUzKnMOcYKWKUCBIqIib1tkpkMNBoQhYxRZQZ2pHLbeCUOe/PdZTSduy6XvFbaLeXv3F8HivBwxCdoqBzrQuD1JWLPaEL3AwT/UhQJOrdtVrnUzCMZ4LgUOqrVND+BsuZXTVGScvD77i4oWIzoIWAMd5oZdgITGO9+xLKssk0SXgWnbNBrTOtDp8LQX0FGzahwmYjKWOvo/BfHfbEf0XRkNGM5KEqwcfXf73uaauu/54C7t1/dBIzNo9I8nXV4AP73261t/K/TqvTadf531PAVv13w+Kgr6MR7vtbkjQiqkhAFOk3AGz1Ns8qPaco65xdG3IKy8nWSQwruPj2FtwrGlKCAXmUD8PnzzgrJFMaSk0HdFbj3qRTDPUUzdBl3HsE7TswMP023seEpp/LaPk25t6HTwaV4zgZyo2mCmJuTt8tWJJuhiTn1vVDngbesk3CZEHaBlmh5ywIW42nNgg3bm8dYDNwfyNhSgt8EeaILKbDCMOztEoDqAz2cSVmfu91qnHv4k+YHCJvCo41Hk2OxoG+fW5zrOGJ4V7/H9Ak5OsIDeWrAsAD/v+409rx/8fHndr/PwWUvRo6TekVQeBNsflfGgWeyfcvaBhhseaZCubPDBq6PtVMC2qKcGlnZX43GzzjKbpZI6XE5b7iop85buUvLkqCH0L0x8sAkB/3jK3SVht8ccyzojofwii5oP6NTKNNgug8Ji0wnLDYD9OAwstEMNTQ39zrjA/3NfI3JmoBza0EtPkKq+1PIBek0ztB8nkMrBrQoTT5JboEyG3C3FOxZD4d+L42gtGjOdA1OsGgLQrJnEcePwtMB/0+NEvWaYa0jXLJ0CTXyH5/57UicxxvVvGM0zAcczTtdcXc7YqkeLnZGy1HFBF0KcWAA94eCRbrhIrSnP32kzdXECNSLM93MvNydOvuR48qfx+VPQHPLvNKuV8FK6s0bZDwR03IT4VA3+gIqh+QovyxpI4No6iaynp3s3ayjn1ZVtQeamwec7wgszY3dDYH8v+kZzFc5ggGxfptyivTbHq8fHbdQ3IFTOpGVWkLKwiz15uMWCepHziLoXnULOOi8bJsSPY8XJwP3pxf/X5+cX6mu4+/jwZvzyfjwdl5MRNgqQn9JHjULw0CzBgNgys6q45m49oN9Quf6BbRrJgrqOSp8KksL9f59FYWXsxDqeIs9253ynIteZhG9K32FHJXQGubJRqRnmjZe7SlN8oUd3zLFik/rwXLEn5pKZhDQGckDdVbHiAazP/quuO54N78X0yJf4A/AjyQ/3c7J9vf/07bp/X3/ycB3d0o1wBmy0mqFlyw/9lPAjc/mNxn0x0KUWdUXPGQfkVl8I3m/CINtct0dDvoZ8HTxIjgQKn9U+37NLaChAO+1Z80D9VgvXfMQ35Ual/ZWFu537xeUjHNaMypMteQSXuz0uWGuUuKuzTBraO7sjSbe5jOvbgsvTNlnH2/x/Gj4vSAqXp0V2svg6ttbjYs7ufLMWWquZkWa++0WrsgyD4VVr50lSckrLR/xYstFRTR31KnSwzi9lZSX9DsnsZBgmlL9lRRmwObhkq+0FQIlQdiywVZthVMiejOwBRPI4vndnwzY+fVBz61NwkPNjdeyOfmIUqV+Ty4otMF5zd+uROZ0USSPMq1gbGbxSx/W9rS5t+bu3uW9R1dSRJzHPZqVq/cRfVVnum11cB36KBQ+CydzbfiHt01ilZ0yak/TlMynX7AE26cosU1qdS9f0rP6LnD5l8G7s3/qh7pizPBh77/6j97Vfu/7S5e6vzvCWDv97+tA1y3f/d62ufeuoPAnvO/tC2Lw/0B/KHzf3Lc2/7/R++kV5//pwDbG7YfMbJecB9o6s59oU9AcXrQTnSULwbua+AqMu+DiSA6+CelhvFwNuJqrP8uim6lsckC4fZzo1HqDGqGqlWQ9Q5bPcY+9MzwnS3OPsxIKHVGY4ulO7E0GrsNyD78+z+GK/3JorHzqb3cMROUS4O6+CN7p9Xpuj23ZQnqzHcsuG5MZ34uX0tTJ6BOO+uNZZ16i9ApcDy3kdRQQw011FBDDTXUUEMNNdRQQw011FBDDTXUUEMNNdRQQw3fBPwBPxwbVwBQAAA= + chart: H4sIAAAAAAAAA+0ba2/bOLKf/SsGxn1oi0p+xfGuD/3gptlucG1ixNkWxeGwoCXaZiuLWlKym2v732/40MtW7LrNJtdW0wKWSM6Dw+E8SCUSfMV8Khwe0VDGxHvfenDb0EYY9Pv6F2HzVz93ekedbr97fKzaO73eoPsA+rcuSQUkOGkB8EBwHu8at6//O4Voe/1PFkTE7jVZBrfEY9/6d3udjfXv9zrHD6B9S/x3wk++/iRir6mQjIdDWHUaJIqy17Y7cNuOT1cNn0pPsCjWzSP4nQZL8JSVwIwLiBcUXhDh05AKuEAzmigzgrG1LKAfYjQtxG2EZEmHsG1yjdU2z/tWzE8CFfvf554757fIY8/+77Z7g439f3Q06NX7/y6g1YITHl0LNl/E8NB7BN1251eYjMYwOQXc3CTUL2Q2YwEjMQWPLyMSXrswCgLQaBIElVSsqO/C1YJJwKEU8DdgHtoU9SEJlR9QfmIUEQ9/JnwWr4mg8NIMeQIrF7roKTwaxUAkhDxGPI4oYs0kUgs1+suzk9NzFExxaLRa+D+lUMEko209GnTdNjxUA5q2q/non4rENU9gSa4VU0iQWZxNwgqE3NW0UQGhR2HN4oWRxlBxFY23lgafxgSHE0SI8G1WHAgktkJrWMRxNGy11uu1S7TELhfzllWabNm5Oii1xfojDKhU2v4rYQJnPL0G9NeIQKYoa0DWesHmgmJfzJXUa8FiFs6fgLQKV2R8JmPBpklcUloqI069OADVhibQHE3gbNKEZ6PJ2eSJIvLm7Or3iz+u4M3o8nJ0fnV2OoGLSzi5OH9+dnV2cY5vv8Ho/C386+z8+ROgTK0kqjMSagYoJlPqRItRtCaUlkRIg4qMqMdmzMOphfOEzCnMOUaKEGcEERVLJtWyShTQV2QCtmQxiXXT1rzcBg6Z8+FcRSllx67byv4vlNtLexyPh7HgQYBOUdC50oUm6spFRegC11KiHwhOibZuwlb5FJyFM0GwKfHiRNBhTuXEYI1xpsXmN1y8pyJvUJOAMT4oZZgITEO1+hKKc5NJFHEbnW2j0plSh8eFoF4MuZhQErMRFanXUfhHh4r4H1M0ZDQjeVuV4OH1X783aNf1313AzvVHJzFj8yWJvq0a3LP+nR72lda/2+526/rvTmCj/nvPQn+oohGu+ysSNZY0Jj6JybABYKq3ua30nKysc7ZtyMksx+JJDCuI/PEjuJc0oAQD8nnaDJ8/46iATGkgFR9QWY37PpliqKdohi7jrQN430CBqd6wSgjFP52jkVub+xA+aVKO41iSuaYyZm7K381Ekq4lkkrregFP/NaqQ4JoQTqaWKZnG4SNxhMThBsfPzrAZuC+JkFCM3pLzBFZSM+WGJ6lURpAqXGImJj5vVWpxk7kT5gcomwxHCk6ih0N/ZQkjT3frAOAjLlAjPQVFRQQKc/1aig9lpkoTNeiuNnIlK5GV5kui6/3Y9uBOfIU1Z1EuSQSM2Y/CW4WxCC46TikdN+b7f8Qdvp/n0YBv16ioXxTANjj//u97mDT/x8d1f7/TqDo1dBpylYWBJ5ni/+1UeCefP+CBkss1lq6gvk7g4aqT5XQguoiXJY8kW084Qm6WT1Liegeurehddyxt3hZmPhtTP3wOQCk292KVVhqTS8MuS2qC0FgQb33MlnmCaJzSFqgJWGhFyQ+hYeRYKihf7hXVg73Gco3JvECmhsJaPMRVtufQC5It3+M7PPYEJT0eDua/BpdAqQ2oZ+pWDGPjjxPGcH5wRKoGp1g0BbZzJwDt58BpoL+EJoF69RNyka5ZGiSKswOt7pjMsf2ZpnOOAmCMUfTLsdwgxFlnaWgz5dLEvr58jjQqpjB4jqiojCm2n7SwxWkiByL4x1rXo46unvawhygiktFwDNorULut0lVMY3U2Qyy/aAavEQI9IyOoOoF+cmn5SzEipk9a2w3x5xch54sKklxYqXjocN5lfEP5MbmIccfVIvJQp18638hP0PhIiUwyvA3Oa/1sdbh8zN4++a1ptMF56mzcZbcp0/tVtw1Tu2sp7mp2V65D0WHsac7gtuN2FYmJ40LOO1mteNqDptfZMnNJxUEUp+FRCqcVrMsos+kOk4sbLTSYtjuvG5RpcQ7zkJA1pvTTWmlmqwg9MZ23UCFhqui0zC+7+Xp6Pnp5Z+nL09P1Enzn+ejV6eT8ejkNBsJsFJcfhN8OSw0AswYDfxLOiu32nYVcoZZ/HOztc3GCip5Ijwqi+iqdtqouLJxOKvQ1lmdbnFeKx4kS/pKRQW5PUFjHgUeSzXQiHewV8vIqAPhCmapTeoLHFFgmgWfMVdZ1OCXX9uFXuQXc49jkXx1Mm4UZ7UVqzbk8NKzhaIWv/ZoIQWfzkgSxK9wqw/hqNuuz62/FHbWfwJL6Fv4EGRP/dfrHm/e/w46g7r+uxNQp1vFGlAvOUniBRfsv+ZK6P0vOo7kp4MB6oyKSx7Qb6gMv9OaTySBcnGOOg58IXgS6Sk4UDj+K5/7NTYChwOe0Z/UL+UUqrKthfLEiekyGVDpOe9GBz61POY01r8Bk+ZhrcpN/RRlT0mES0e359JsVgidel1Z6NOZjumvcNSoONWgsxt1qlkp4HpTmlzEarkcfUyhH6YZ7o1WaxB8e1VcuuksDohYYf2yjg0VZBmB4U5XGNjNo6SeoPaZhn6EqYx9K6nNgfxALUXUKWDphZhyURZtBZMtutUwxd3Iwrlpz0dsdb3jU/MQcT9/aAV8rl+WSayvh20m4BVPoi1PZMmXqTYw1rKQpb2FJW0+bm6vmT13diWJ9Hao1KzC3Cb1TZ7pmdHAT+igcPI2xU2XYofuGtlVRMGpH6YpmUzf4Q7XTtHQmpTOPf6WM8P7Dps/DOzM/8oe6aszwX33/+pjv/L5f6eHP3X+dwdQef+7sYHr4/9KT3vfS3crsHv/m8tYfZP7DXXg3u//B0eb93+D/lG9/+8Civs/vXvfTKsmpv1EmUGjfEEUHvwlgDY4xY+KIZS3m6cOy0SDBAFfv9aHSacfIhIa4bAcoo2ICCSjiqch/Ps/9627HwEq9v/KrOTt/QHQvu+/usdb3//1+4N6/98FmLtBc4lt7wKHQBN37gm1JbPoiXaisvysYdcFXkzmQ9ARRCX/UeHC8Gx2zuOx+nMBTCsaeRUIHz83GoU7B/spVnZxZpzNxr3PEPrYXD4s2TEQh950PzWEGQmkKnzMmcqNVBqN7RuQ7HwkvyTMZ9LYuOjYHrzVQD/gQuBUtjrM90xaT+oSvbH18VfxzF1Qbq6usj+t6ra7PbfvmlN8XYuPBVdXpTbzSnFp4vjU6djTdXt3bAg6BRr5V2Ib34gVvhArn4Q5M6INQg/KvgPr9l8w3Vj+viv/uqvZhset7hE8Vv+a971faqihhhpqqKGGGmqooYYaaqihhhpqqKGGGr4X+B8Ff7QvAFAAAA== values: image: tag: 0.7.0-dev diff --git a/controllers/provider-openstack/pkg/apis/config/types.go b/controllers/provider-openstack/pkg/apis/config/types.go index a956a5d95..a6392473d 100644 --- a/controllers/provider-openstack/pkg/apis/config/types.go +++ b/controllers/provider-openstack/pkg/apis/config/types.go @@ -15,6 +15,7 @@ package config import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,9 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to OpenStack-specific identifiers. MachineImages []MachineImage + + // ETCD is the etcd configuration. + ETCD ETCD } // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers. @@ -46,3 +50,25 @@ type CloudProfileMapping struct { // Image is the name of the image. Image string } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + ClassName *string + // Capacity is the storage capacity used in etcd-main volume claims. + Capacity *resource.Quantity +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + Schedule *string +} diff --git a/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go b/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go index 2697e233a..69ac79213 100644 --- a/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go +++ b/controllers/provider-openstack/pkg/apis/config/v1alpha1/types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,9 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to OpenStack-specific identifiers. MachineImages []MachineImage `json:"machineImages,omitempty"` + + // ETCD is the etcd configuration. + ETCD ETCD `json:"etcd"` } // MachineImage is a mapping from logical names and versions to OpenStack-specific identifiers. @@ -46,3 +50,28 @@ type CloudProfileMapping struct { // Image is the name of the image. Image string `json:"image"` } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage `json:"storage"` + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup `json:"backup"` +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + // +optional + ClassName *string `json:"className,omitempty"` + // Capacity is the storage capacity used in etcd-main volume claims. + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty"` +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + // +optional + Schedule *string `json:"schedule,omitempty"` +} diff --git a/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.conversion.go b/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.conversion.go index f024ab2eb..8fd7a3d5e 100644 --- a/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.conversion.go +++ b/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.conversion.go @@ -24,6 +24,7 @@ import ( unsafe "unsafe" config "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + resource "k8s.io/apimachinery/pkg/api/resource" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -55,6 +56,36 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ETCD)(nil), (*config.ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCD_To_config_ETCD(a.(*ETCD), b.(*config.ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCD)(nil), (*ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCD_To_v1alpha1_ETCD(a.(*config.ETCD), b.(*ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDBackup)(nil), (*config.ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(a.(*ETCDBackup), b.(*config.ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDBackup)(nil), (*ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(a.(*config.ETCDBackup), b.(*ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDStorage)(nil), (*config.ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(a.(*ETCDStorage), b.(*config.ETCDStorage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDStorage)(nil), (*ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(a.(*config.ETCDStorage), b.(*ETCDStorage), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MachineImage)(nil), (*config.MachineImage)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_MachineImage_To_config_MachineImage(a.(*MachineImage), b.(*config.MachineImage), scope) }); err != nil { @@ -92,6 +123,9 @@ func Convert_config_CloudProfileMapping_To_v1alpha1_CloudProfileMapping(in *conf func autoConvert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration(in *ControllerConfiguration, out *config.ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]config.MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_v1alpha1_ETCD_To_config_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -102,6 +136,9 @@ func Convert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration( func autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in *config.ControllerConfiguration, out *ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_config_ETCD_To_v1alpha1_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -110,6 +147,78 @@ func Convert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration( return autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in, out, s) } +func autoConvert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + if err := Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_ETCD_To_config_ETCD is an autogenerated conversion function. +func Convert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCD_To_config_ETCD(in, out, s) +} + +func autoConvert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + if err := Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_config_ETCD_To_v1alpha1_ETCD is an autogenerated conversion function. +func Convert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + return autoConvert_config_ETCD_To_v1alpha1_ETCD(in, out, s) +} + +func autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup is an autogenerated conversion function. +func Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in, out, s) +} + +func autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup is an autogenerated conversion function. +func Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + return autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in, out, s) +} + +func autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage is an autogenerated conversion function. +func Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in, out, s) +} + +func autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage is an autogenerated conversion function. +func Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + return autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in, out, s) +} + func autoConvert_v1alpha1_MachineImage_To_config_MachineImage(in *MachineImage, out *config.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 6f2800b54..220df1008 100644 --- a/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/controllers/provider-openstack/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -72,6 +73,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-openstack/pkg/apis/config/zz_generated.deepcopy.go b/controllers/provider-openstack/pkg/apis/config/zz_generated.deepcopy.go index f07c4c73e..578567085 100644 --- a/controllers/provider-openstack/pkg/apis/config/zz_generated.deepcopy.go +++ b/controllers/provider-openstack/pkg/apis/config/zz_generated.deepcopy.go @@ -51,6 +51,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -72,6 +73,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-openstack/pkg/cmd/config.go b/controllers/provider-openstack/pkg/cmd/config.go index ba8043da7..bcfb659f7 100644 --- a/controllers/provider-openstack/pkg/cmd/config.go +++ b/controllers/provider-openstack/pkg/cmd/config.go @@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) { *machineImages = c.Config.MachineImages } +// ApplyETCDStorage sets the given etcd storage configuration to that of this Config. +func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { + *etcdStorage = c.Config.ETCD.Storage +} + +// ApplyETCDBackup sets the given etcd backup configuration to that of this Config. +func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) { + *etcdBackup = c.Config.ETCD.Backup +} + // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/controllers/provider-openstack/pkg/cmd/options.go b/controllers/provider-openstack/pkg/cmd/options.go index 94997f1ed..9d08eb47b 100644 --- a/controllers/provider-openstack/pkg/cmd/options.go +++ b/controllers/provider-openstack/pkg/cmd/options.go @@ -18,10 +18,16 @@ import ( "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/controlplane" infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/controller/worker" + controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplane" + controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplanebackup" + controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/webhook/controlplaneexposure" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane" extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" extensionsworkercontroller "github.com/gardener/gardener-extensions/pkg/controller/worker" + + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" + extensioncontrolplanewebhook "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" ) // ControllerSwitchOptions are the controllercmd.SwitchOptions for the provider controllers. @@ -32,3 +38,12 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { controllercmd.Switch(extensionsworkercontroller.ControllerName, workercontroller.AddToManager), ) } + +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions( + webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager), + ) +} diff --git a/controllers/provider-openstack/pkg/openstack/types.go b/controllers/provider-openstack/pkg/openstack/types.go index 0d052ac5a..6d388cfd3 100644 --- a/controllers/provider-openstack/pkg/openstack/types.go +++ b/controllers/provider-openstack/pkg/openstack/types.go @@ -19,18 +19,18 @@ import "path/filepath" const ( // Name is the name of the OpenStack provider. Name = "provider-openstack" + // StorageProviderName is the name of the Openstack storage provider. + StorageProviderName = "Swift" // MachineControllerManagerImageName is the name of the MachineControllerManager image. MachineControllerManagerImageName = "machine-controller-manager" // HyperkubeImageName is the name of the hyperkube image. HyperkubeImageName = "hyperkube" + // ETCDBackupRestoreImageName is the name of the etcd backup and restore image. + ETCDBackupRestoreImageName = "etcd-backup-restore" // AuthURL is a constant for the key in a cloud provider secret that holds the OpenStack auth url. AuthURL = "authURL" - - // CloudProviderConfigName is the name of the configmap containing the cloud provider config. - CloudProviderConfigName = "cloud-provider-config" - // DomainName is a constant for the key in a cloud provider secret that holds the OpenStack domain name. DomainName = "domainName" // TenantName is a constant for the key in a cloud provider secret that holds the OpenStack tenant name. @@ -40,8 +40,18 @@ const ( // Password is a constant for the key in a cloud provider secret and backup secret that holds the OpenStack password. Password = "password" + // BucketName is a constant for the key in a backup secret that holds the bucket name. + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + BucketName = "bucketName" + + // CloudProviderConfigName is the name of the configmap containing the cloud provider config. + CloudProviderConfigName = "cloud-provider-config" // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" + // BackupSecretName defines the name of the secret containing the credentials which are required to + // authenticate against the respective cloud provider (required to store the backups of Shoot clusters). + BackupSecretName = "etcd-backup" ) var ( diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/app.go b/controllers/provider-openstack/pkg/webhook/controlplane/app.go new file mode 100644 index 000000000..5fcb09fa1 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/app.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = log.Log.WithName("openstack-controlplane-webhook") + +// AddToManager creates a webhook and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.ShootKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(logger), controlplane.NewUnitSerializer(), + controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), + }) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go new file mode 100644 index 000000000..faf92b5df --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer.go @@ -0,0 +1,145 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/coreos/go-systemd/unit" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// NewEnsurer creates a new controlplane ensurer. +func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + logger: logger.WithName("openstack-controlplane-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + logger logr.Logger +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-apiserver"); c != nil { + ensureKubeAPIServerCommandLineArgs(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +// EnsureKubeControllerManagerDeployment ensures that the kube-controller-manager deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeControllerManagerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-controller-manager"); c != nil { + ensureKubeControllerManagerCommandLineArgs(c) + ensureVolumeMounts(c) + } + ensureVolumes(ps) + return nil +} + +func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "openstack") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = controlplane.EnsureNoStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") +} + +func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-config=", + "/etc/kubernetes/cloudprovider/cloudprovider.conf") + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-cloud-volume-plugin=", "openstack") +} + +var ( + cloudProviderConfigVolumeMount = corev1.VolumeMount{ + Name: openstack.CloudProviderConfigName, + MountPath: "/etc/kubernetes/cloudprovider", + } + cloudProviderSecretVolumeMount = corev1.VolumeMount{ + Name: common.CloudProviderSecretName, + MountPath: "/srv/cloudprovider", + } + + cloudProviderConfigVolume = corev1.Volume{ + Name: openstack.CloudProviderConfigName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.CloudProviderConfigName}, + }, + }, + } + cloudProviderSecretVolume = corev1.Volume{ + Name: common.CloudProviderSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + SecretName: common.CloudProviderSecretName, + }, + }, + } +) + +func ensureVolumeMounts(c *corev1.Container) { + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderConfigVolumeMount) + c.VolumeMounts = controlplane.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderSecretVolumeMount) +} + +func ensureVolumes(ps *corev1.PodSpec) { + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderConfigVolume) + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, cloudProviderSecretVolume) +} + +// EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. +func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, opts []*unit.UnitOption) ([]*unit.UnitOption, error) { + if opt := controlplane.UnitOptionWithSectionAndName(opts, "Service", "ExecStart"); opt != nil { + command := controlplane.DeserializeCommandLine(opt.Value) + command = ensureKubeletCommandLineArgs(command) + opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n ") + } + return opts, nil +} + +func ensureKubeletCommandLineArgs(command []string) []string { + command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "openstack") + return command +} + +// EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. +func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { + // Make sure CSI-related feature gates are not enabled + // TODO Leaving these enabled shouldn't do any harm, perhaps remove this code when properly tested? + delete(kubeletConfig.FeatureGates, "VolumeSnapshotDataSource") + delete(kubeletConfig.FeatureGates, "CSINodeInfo") + delete(kubeletConfig.FeatureGates, "CSIDriverRegistry") + return nil +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go new file mode 100644 index 000000000..80472d560 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplane/ensurer_test.go @@ -0,0 +1,286 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test" + + "github.com/coreos/go-systemd/unit" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + ctrl *gomock.Controller + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--enable-admission-plugins=Priority,NamespaceLifecycle", + "--disable-admission-plugins=PersistentVolumeLabel", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: openstack.CloudProviderConfigName, MountPath: "?"}, + // TODO Use constant from github.com/gardener/gardener/pkg/apis/core/v1alpha1 when available + // See https://github.com/gardener/gardener/pull/930 + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: openstack.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureKubeControllerManagerDeployment", func() { + It("should add missing elements to kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + Command: []string{ + "--cloud-provider=?", + "--cloud-config=?", + "--external-cloud-volume-plugin=?", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: openstack.CloudProviderConfigName, MountPath: "?"}, + {Name: common.CloudProviderSecretName, MountPath: "?"}, + }, + }, + }, + Volumes: []corev1.Volume{ + {Name: openstack.CloudProviderConfigName}, + {Name: common.CloudProviderSecretName}, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + }) + + Describe("#EnsureKubeletServiceUnitOptions", func() { + It("should modify existing elements of kubelet.service unit options", func() { + var ( + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --cloud-provider=openstack`, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletServiceUnitOptions method and check the result + opts, err := ensurer.EnsureKubeletServiceUnitOptions(context.TODO(), oldUnitOptions) + Expect(err).To(Not(HaveOccurred())) + Expect(opts).To(Equal(newUnitOptions)) + }) + }) + + Describe("#EnsureKubeletConfiguration", func() { + It("should modify existing elements of kubelet configuration", func() { + var ( + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletConfiguration method and check the result + kubeletConfig := *oldKubeletConfig + err := ensurer.EnsureKubeletConfiguration(context.TODO(), &kubeletConfig) + Expect(err).To(Not(HaveOccurred())) + Expect(&kubeletConfig).To(Equal(newKubeletConfig)) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=openstack")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ",")) + Expect(c.Command).To(Not(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ","))) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} + +func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { + // Check that the kube-controller-manager container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=external")) + Expect(c.Command).To(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) + Expect(c.Command).To(ContainElement("--external-cloud-volume-plugin=openstack")) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderConfigVolumeMount)) + Expect(c.VolumeMounts).To(ContainElement(cloudProviderSecretVolumeMount)) + + // Check that the Pod spec contains all needed volumes + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderSecretVolume)) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go new file mode 100644 index 000000000..cb3f51dc7 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/imagevector" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the Openstack backup webhook to the manager. +type AddOptions struct { + // ETCDBackup is the etcd backup configuration. + ETCDBackup config.ETCDBackup +} + +var logger = log.Log.WithName("openstack-controlplanebackup-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.BackupKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDBackup, imagevector.ImageVector(), logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go new file mode 100644 index 000000000..141004afd --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer.go @@ -0,0 +1,153 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdBackup *config.ETCDBackup, imageVector imagevector.ImageVector, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdBackup: etcdBackup, + imageVector: imageVector, + logger: logger.WithName("openstack-controlplanebackup-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdBackup *config.ETCDBackup + imageVector imagevector.ImageVector + logger logr.Logger +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + if err := e.ensureContainers(&ss.Spec.Template.Spec, ss.Name, cluster); err != nil { + return err + } + return nil +} + +func (e *ensurer) ensureContainers(ps *corev1.PodSpec, name string, cluster *extensionscontroller.Cluster) error { + c, err := e.getBackupRestoreContainer(name, cluster) + if err != nil { + return err + } + ps.Containers = controlplane.EnsureContainerWithName(ps.Containers, *c) + return nil +} + +func (e *ensurer) getBackupRestoreContainer(name string, cluster *extensionscontroller.Cluster) (*corev1.Container, error) { + // Find etcd-backup-restore image + // TODO Get seed version from clientset when it's possible to inject it + image, err := e.imageVector.FindImage(openstack.ETCDBackupRestoreImageName, "", cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, errors.Wrapf(err, "could not find image %s", openstack.ETCDBackupRestoreImageName) + } + + const ( + defaultSchedule = "0 */24 * * *" + ) + + // Determine provider, container env variables, and volume mounts + // They are only specified for the etcd-main stateful set (backup is enabled) + var ( + provider string + env []corev1.EnvVar + ) + if name == common.EtcdMainStatefulSetName { + provider = openstack.StorageProviderName + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: openstack.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + }, + }, + }, + { + Name: "OS_AUTH_URL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.AuthURL, + }, + }, + }, + { + Name: "OS_DOMAIN_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.DomainName, + }, + }, + }, + { + Name: "OS_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.UserName, + }, + }, + }, + { + Name: "OS_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.Password, + }, + }, + }, + { + Name: "OS_TENANT_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.TenantName, + }, + }, + }, + } + } + + // Determine schedule + var schedule = defaultSchedule + if e.etcdBackup.Schedule != nil { + schedule = defaultSchedule + } + + return controlplane.GetBackupRestoreContainer(name, schedule, provider, image.String(), nil, env, nil), nil +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go new file mode 100644 index 000000000..710928610 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplanebackup/ensurer_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Backup Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + Describe("#EnsureETCDStatefulSet", func() { + var ( + etcdBackup = &config.ETCDBackup{ + Schedule: util.StringPtr("0 */24 * * *"), + } + + imageVector = imagevector.ImageVector{ + { + Name: openstack.ETCDBackupRestoreImageName, + Repository: "test-repository", + Tag: "test-tag", + }, + } + + cluster = &extensionscontroller.Cluster{ + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + ) + + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + var ( + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: openstack.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + }, + }, + }, + { + Name: "OS_AUTH_URL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.AuthURL, + }, + }, + }, + { + Name: "OS_DOMAIN_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.DomainName, + }, + }, + }, + { + Name: "OS_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.UserName, + }, + }, + }, + { + Name: "OS_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.Password, + }, + }, + }, + { + Name: "OS_TENANT_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: openstack.BackupSecretName}, + Key: openstack.TenantName, + }, + }, + }, + } + ) + + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdMainStatefulSetName, "0 */24 * * *", openstack.StorageProviderName, + "test-repository:test-tag", nil, env, nil))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdEventsStatefulSetName, "0 */24 * * *", "", + "test-repository:test-tag", nil, nil, nil))) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go new file mode 100644 index 000000000..cb7559241 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/openstack" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the Openstack exposure webhook to the manager. +type AddOptions struct { + // ETCDStorage is the etcd storage configuration. + ETCDStorage config.ETCDStorage +} + +var logger = log.Log.WithName("openstack-controlplaneexposure-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.SeedKind, + Provider: openstack.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &corev1.Service{}, &appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDStorage, logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go new file mode 100644 index 000000000..8f4704493 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer.go @@ -0,0 +1,85 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdStorage *config.ETCDStorage, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdStorage: etcdStorage, + logger: logger.WithName("openstack-controlplaneexposure-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdStorage *config.ETCDStorage + client client.Client + logger logr.Logger +} + +// InjectClient injects the given client into the ensurer. +func (m *ensurer) InjectClient(client client.Client) error { + m.client = client + return nil +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + // Get load balancer address of the kube-apiserver service + address, err := controlplane.GetLoadBalancerIngress(ctx, e.client, dep.Namespace, common.KubeAPIServerDeploymentName) + if err != nil { + return errors.Wrap(err, "could not get kube-apiserver service load balancer address") + } + + if c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver"); c != nil { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--advertise-address=", address) + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-hostname=", address) + } + return nil +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + e.ensureVolumeClaimTemplates(&ss.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureVolumeClaimTemplates(spec *appsv1.StatefulSetSpec, name string) { + t := e.getVolumeClaimTemplate(name) + spec.VolumeClaimTemplates = controlplane.EnsurePVCWithName(spec.VolumeClaimTemplates, *t) +} + +func (e *ensurer) getVolumeClaimTemplate(name string) *corev1.PersistentVolumeClaim { + var etcdStorage config.ETCDStorage + if name == common.EtcdMainStatefulSetName { + etcdStorage = *e.etcdStorage + } + return controlplane.GetETCDVolumeClaimTemplate(name, etcdStorage.ClassName, etcdStorage.Capacity) +} diff --git a/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go new file mode 100644 index 000000000..40fbe9689 --- /dev/null +++ b/controllers/provider-openstack/pkg/webhook/controlplaneexposure/ensurer_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-openstack/pkg/apis/config" + mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openstack Controlplane Exposure Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + etcdStorage = &config.ETCDStorage{ + ClassName: util.StringPtr("gardener.cloud-fast"), + Capacity: util.QuantityPtr(resource.MustParse("25Gi")), + } + + ctrl *gomock.Controller + + svcKey = client.ObjectKey{Namespace: namespace, Name: common.KubeAPIServerDeploymentName} + svc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "1.2.3.4"}, + }, + }, + }, + } + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{"--advertise-address=?", "--external-hostname=?"}, + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureETCDStatefulSet", func() { + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-main"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-events"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--advertise-address=1.2.3.4")) + Expect(c.Command).To(ContainElement("--external-hostname=1.2.3.4")) +} + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-main") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdMainStatefulSetName, util.StringPtr("gardener.cloud-fast"), + util.QuantityPtr(resource.MustParse("25Gi"))))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-events") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdEventsStatefulSetName, nil, nil))) +} + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *corev1.Service: + *obj.(*corev1.Service) = *result.(*corev1.Service) + } + return nil + } +}