diff --git a/Makefile b/Makefile index dca12255e..e5a6df97b 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,10 @@ start-provider-alicloud: ./controllers/provider-alicloud/cmd/gardener-extension-provider-alicloud \ --config-file=./controllers/provider-alicloud/example/00-componentconfig.yaml \ --infrastructure-ignore-operation-annotation=$(IGNORE_OPERATION_ANNOTATION) \ - --leader-election=$(LEADER_ELECTION) + --leader-election=$(LEADER_ELECTION) \ + --webhook-config-mode=url \ + --webhook-config-name=alicloud-webhooks \ + --webhook-config-host=$(HOSTNAME) .PHONY: start-provider-packet start-provider-packet: diff --git a/controllers/provider-alicloud/charts/provider-alicloud/templates/configmap.yaml b/controllers/provider-alicloud/charts/provider-alicloud/templates/configmap.yaml index 9a243bb21..e2b5eab2d 100644 --- a/controllers/provider-alicloud/charts/provider-alicloud/templates/configmap.yaml +++ b/controllers/provider-alicloud/charts/provider-alicloud/templates/configmap.yaml @@ -15,3 +15,9 @@ data: machineImages: {{ toYaml .Values.config.machineImages | indent 4 }} {{- end }} + etcd: + storage: + className: {{ .Values.config.etcd.storage.className }} + capacity: {{ .Values.config.etcd.storage.capacity }} + backup: + schedule: {{ .Values.config.etcd.backup.schedule }} diff --git a/controllers/provider-alicloud/charts/provider-alicloud/templates/storageclass.yaml b/controllers/provider-alicloud/charts/provider-alicloud/templates/storageclass.yaml new file mode 100644 index 000000000..c05ee8b2f --- /dev/null +++ b/controllers/provider-alicloud/charts/provider-alicloud/templates/storageclass.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.config.etcd.storage.className }} +provisioner: diskplugin.csi.alibabacloud.com +allowVolumeExpansion: true +parameters: + csi.storage.k8s.io/fstype: ext4 + type: cloud_ssd + readOnly: false diff --git a/controllers/provider-alicloud/charts/provider-alicloud/values.yaml b/controllers/provider-alicloud/charts/provider-alicloud/values.yaml index 2135bd4bf..940d87eb8 100644 --- a/controllers/provider-alicloud/charts/provider-alicloud/values.yaml +++ b/controllers/provider-alicloud/charts/provider-alicloud/values.yaml @@ -6,16 +6,29 @@ image: resources: {} controllers: + controlplane: + concurrentSyncs: 5 infrastructure: concurrentSyncs: 5 ignoreOperationAnnotation: false worker: concurrentSyncs: 5 -disableControllers: [] +disableControllers: +- controlplane-controller +disableWebhooks: +- controlplane +- controlplaneexposure +- controlplanebackup config: machineImages: - name: coreos version: 2023.5.0 id: coreos_2023_4_0_64_30G_alibase_20190319.vhd + etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-alicloud/cmd/gardener-extension-provider-alicloud/app/app.go b/controllers/provider-alicloud/cmd/gardener-extension-provider-alicloud/app/app.go index 01108700b..395e513ce 100644 --- a/controllers/provider-alicloud/cmd/gardener-extension-provider-alicloud/app/app.go +++ b/controllers/provider-alicloud/cmd/gardener-extension-provider-alicloud/app/app.go @@ -25,9 +25,12 @@ import ( alicloudcontrolplane "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/controlplane" alicloudinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/infrastructure" alicloudworker "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/worker" + alicloudcontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/webhook/controlplanebackup" + alicloudcontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/webhook/controlplaneexposure" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -63,7 +66,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { MaxConcurrentReconciles: 5, } - controllerSwitches = alicloudcmd.ControllerSwitchOptions() + controllerSwitches = alicloudcmd.ControllerSwitchOptions() + webhookSwitches = alicloudcmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("alicloud-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, @@ -73,6 +87,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.PrefixOption("worker-", workerCtrlOpts), configFileOpts, controllerSwitches, + webhookOptions, ) ) @@ -98,6 +113,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts.Completed().ApplyMachineImages(&alicloudworker.DefaultAddOptions.MachineImages) + configFileOpts.Completed().ApplyETCDStorage(&alicloudcontrolplaneexposure.DefaultAddOptions.ETCDStorage) + configFileOpts.Completed().ApplyETCDBackup(&alicloudcontrolplanebackup.DefaultAddOptions.ETCDBackup) controlPlaneCtrlOpts.Completed().Apply(&alicloudcontrolplane.Options) infraCtrlOpts.Completed().Apply(&alicloudinfrastructure.DefaultAddOptions.Controller) infraReconcileOpts.Completed().Apply(&alicloudinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation) @@ -107,6 +124,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { + controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") + } + if err := mgr.Start(ctx.Done()); err != nil { controllercmd.LogErrAndExit(err, "Error running manager") } diff --git a/controllers/provider-alicloud/example/00-componentconfig.yaml b/controllers/provider-alicloud/example/00-componentconfig.yaml index a3db3a865..6e11754b7 100644 --- a/controllers/provider-alicloud/example/00-componentconfig.yaml +++ b/controllers/provider-alicloud/example/00-componentconfig.yaml @@ -5,3 +5,9 @@ machineImages: - name: coreos version: 2023.5.0 id: coreos_2023_4_0_64_30G_alibase_20190319.vhd +etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-alicloud/example/30-etcd-backup-secret.yaml b/controllers/provider-alicloud/example/30-etcd-backup-secret.yaml new file mode 100644 index 000000000..8c082591c --- /dev/null +++ b/controllers/provider-alicloud/example/30-etcd-backup-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-backup + namespace: shoot--foo--bar +type: Opaque +data: + bucketName: YnVja2V0MTIz # bucket123 + storageEndpoint: ZXUtd2VzdC0x # eu-west-1 + accessKeyID: YWRtaW4= # admin + accessKeySecret: YWRtaW4= # admin diff --git a/controllers/provider-alicloud/example/controller-registration.yaml b/controllers/provider-alicloud/example/controller-registration.yaml index b223c9f18..30f892c68 100644 --- a/controllers/provider-alicloud/example/controller-registration.yaml +++ b/controllers/provider-alicloud/example/controller-registration.yaml @@ -14,7 +14,7 @@ spec: deployment: type: helm providerConfig: - chart: H4sIAAAAAAAAA+0aa2/bOLKf/SsGxn1oD5HkZ3zrw35w02zXuNQJ4uwWxeEQ0BIts5FFHUnZ9aX97zcUJVmynGSTuuntVQMDlihyXhzOS4oEXzGPCosEzA147DkvDg4thEG/n/wj7P4n1+1ur93pd46P9Xi72x4MXkD/8KxUIZaKCIAXgnN137yHnv9JIars/8mCCGVvyDI4FI2H9r/TGezsf7/dbb2A1qEYuA9+8P0nEfudCsl4OIRVu0GiKL9t2QO7ZXl01fCodAWLVDI8gl9psARXmwnMuQC1oPCWCI+GVMAoNSO4SA0L6CdFQ42xEZIlHULF4hqrKsXvrZYfBqrn3+Ou7fND0njg/Hda3e7O+e91Br36/D8HOA6c8GgjmL9Q8NJ9BZ1W+yeYji5gegp4uEmY3JD5nAWMKAouX0Yk3Nh40gNIlkkQVFKxop4NVwsmAadSwH+0KDz51IM41I5A+4lRRFz8m/K5WhNB4cxMOYKVDR10FS6NFBAJIVe4juMSsWYSsYXJ8rPxyekEGdMUGo6DvwzDHiI57tSjQcduwUs9oZk+ar76u0ax4TEsyUYThRiJqVyIlCGkrsVGBYQuhTVTC8ONwWJrHB9SHHymCE4nuCDCu3lxIhCVMp3AQqlo6Djr9domCcc2F76TKk06qawWcp2u+i0MqNTa/nfMBEo82wD6a1xAZshrQNbJhvmC4jPFNddrwRQL/SOQqcI1Go9JJdgsViWlZTyi6MUJqDY0geZoCuNpE16PpuPpkUbyfnz16/lvV/B+dHk5mlyNT6dwfgkn55M346vx+QTvfoHR5AP8Yzx5cwSU6Z1EdUZCS4BsMq1OtBiNa0ppiYUsqMiIumzOXBQt9GPiU/A5xooQJYKIiiWTelslMuhpNAFbMkVUMlSRy27gFJ8PfR2ltB3btpP/FsS9cbInlstDJXgQoFMU1Ne6SJDaclGNXWCniOgnghJR567FOp+CcTgXBIdiV8WCDnMkJ2bRBcpZGH3PxQ0V+b0WAC6QU60IE31pqHdeQlEuGUcRTyNzOqj1pVXhciGoq2DLI5R4bERF7HUM/hGgGv8VRUNGS5IHqwQfX//124N+Xf89B9y3/+gn5sxfkugrq8EH9r+N+V55/zuYFHbr/O85YKf+u2GhN9TxCDf+HYkaS6qIRxQZNgBM+eanlZ6V13VWxYas3HLSZRIjC669vQX7kgaUYDyeZMPw5QvOCsiMBlKTAZ3U2DfxDCM9RTO0GXf+OOk7ELAQtzncx4Mmn0lo2E6sfQifE1SWZaUot3rKaNkZdTtnSNopjoxX2xyqVZsE0YK0E1y5ktMobNQdmyjcuL21gM3B/p0EMc3xLTFBZCEdLzE+S6MygNLgEFdi2vdBJxr3Lv6MmSHypqCn8WhyNPT05fe2xRqeH+7z/x6NAr5Zoql8XQB4wP/32oPurv/v9Nu1/38OKPm1KJJOHgTe5Lv/xCjwfXz/ggZLLNWcpIb5hjFDF6eaZUGTClyaWanfTQdPeIxuNpFR4nJXcTFMHbdyF2cFsQ8g+ONFAMjOespVYZsTfGHI04I6G8IYuaDujYyX2+zQekRKkDDCQjeIPQovI8FQP3+xr1I27NfI3gVRC2juJJ/NV1hpfwa5IJ3+MVLPImDZeA6kx6doEiAziOSaihVz6ch1tQVMHsuALtAJBmyRy2U97twZYDreD6FZMMxkSJsnlwytcYPMDyuPFfFxvFnGcxEHwQVHEpuSpZsVUf5wuzFajOWSoCvJByxw9giw2ERUFObstZ2sqYIIkWBxupWalqU7dj87VLn7iFTjnFnlFHK+ElJW6tUg3U+ajhsLgR7RElTfIEH5c0EZWz5RMaX19nbtdBO6sqimPdSYH3L84xE1SaG1PYp/kJ7BcJ4hGOXrdymvky7T4+Uz6x6Sy2NSt6gKO1hCmD7epsI6O/3IWQjNo2YRFw1XRTMyh+HsdPTm9PL69Oz0RPccryejd6fTi9HJaT4TYKUJ/SL4clgYBJgzGniXdF4eTce1Bxrm3tDOg1g+V1DJY+FSWVyuE+md9Dufh1KFadLd7hTlWvEgXtJ32kvIqoDGNgs0lnqiYe+xht4oEqz4lR1KblYAFgV8Yv2XgUfnJA7UO+4hll6nVZcb/wtwX/4vZsQ9xIcAD+T/3XZ/9/3f8UD3/+r8/9uD7m8Ua4Bkz0msFlyw/5jXAjd/SzKgbXcoQJ1RcckD+vTK4E+Z84s40K7T0t2gt4LHUSKABYX2T7nv09iJFRa4RnkyuSnH7L1jDvKjYvPIhNzS9fbxiopZSsOnKvkPmDQXa11uJFdRfhVHuG+0KkuzuYfpzJ3LwrOkiDPPqwEA9abvk6JHN7X28rfeZWbL4X62rKRETS5m+do7LTZdwAq7kz/YETAP8QY5XWGkNpeSuoKm1zT0IsxN0ruSUizYNkuyhUkJULohph6QRUvAvIdWBmZ40Fjom/HtjMqjj3xmLiLubS+cgPsGIyLky0xWDMAsZEk5V96P5l+bVYWnPUNbksg0O/fpTa+sovoql/LayPejeRaUPE1Hs324R3GNvIdccMWPUpOMZx/xaCbOzKCalkrWb9Ho+d6hroY9cF/+V/ZaT88EH3r/2xvsvP/ttNutTp3/PQfsff+34wvq9m/VYX/vfTsUVM//yvQuDvgB+EPn/7jT2/3+o3s8qM//c4DpEZv3GGlPeAg0tn1X6EOQnx+0E50w5AP3NXIV8YeQhBCdR0SFxvF4PuHqQn8uim6lsU0m4fZLo1HoEWqGyoWQcQ873cYh9JPhO5udQ5iTQOrcyNRLd2JpVDuRQ/jnvxKm9GuLRuVle7F3JiiXCeb8S/ZOq9O1+3bLsOdlk671+HXvunV93Lvutt5e46mboVu51l/dtrrtn+zVwvu/8S011FBDDTXUUEMNNdRQQw011FBDDTXUUEMNNdRQQw011FDD88J/AcPuY4UAUAAA + chart: H4sIAAAAAAAAA+0bXXPbOK7P/hUYzz20nUr+Tm59sw9umu16rnUycbadPmVoibbZyKKOpOz42v73A0lJlizH3rRuunsV2hlLFAECIIgPkokEXzKfCocEzAt47DeeHB2aCKe9nvlF2P41z61Ot9XutU9OdHur0zo9fQK947NShlgqIgCeCM7Vvn6Hvv9NISrN/9mcCOWuySI41hiH5r/dPt2a/16r03wCzWMxsA9+8vknEXtHhWQ87MOyVSNRlL023VO36fh0WfOp9ASLlGkewO80WICnzQSmXICaU3hNhE9DKmCQmBFcJoYF9E7RUFOshWRB+1CyuNqyPOKPVstPA+X173PPnfFjjnFg/bebnc7W+u+2T7vV+n8MaDTgjEdrwWZzBU+9Z9Butn6B8eASxueAi5uE5oVMpyxgRFHw+CIi4drFlR6AQZMgqKRiSX0XrudMAnalgL9oUbjyqQ9xqB2B9hODiHj4M+ZTtSKCwhvb5QUsXWijq/BopIBICLlCPI4oYsUkUgsN+pvh2fkIGdMj1BoN/J9S2DFIRjvxaNB2m/BUd6gnn+rP/qVJrHkMC7LWg0KMg6lMiIQhHF2LjQoIPQorpuaWG0vF1TQ+JDT4RBHsThAhwrdpviMQlTBtYK5U1G80VquVSwzHLhezRqI02UhkdZDrBOuPMKBSa/s/MRMo8WQN6K8RgUyQ14CszITNBMVvimuuV4IpFs5egEwUrsn4TCrBJrEqKC3lEUXPd0C1oQnUB2MYjuvwcjAejl9oIu+H179f/HEN7wdXV4PR9fB8DBdXcHYxejW8Hl6M8O03GIw+wL+Ho1cvgDI9k6jOSGgJkE2m1YkWo2mNKS2wkAYVGVGPTZmHooWzmMwozDjGihAlgoiKBZN6WiUy6GsyAVswRZRpKsnl1rDLjPdnOkppO3bdRvZ/TrzbRvrF8XioBA8CdIqCzrQuDFFXzsuxC9yEEL0jKBFt3Ies8ykYhlNBsCn2VCxoPyNyZpEuUc5c63subqnI3rUAcImcakXY6EtDPfMS8nLJOIp4EpmTRq0vrQqPC0E9BRseocBjLcpTr2LwzwDl+K8oGjJakjxaJfjw+q/XOu1V9d9jwL75Rz8xZbMFib6xGjww/61Obyv/azebnVaV/z0GbNV/tyz0+zoe4cS/JVFtQRXxiSL9GoAt32ZJpedkdZ1TsiEns5wETWJkQdxPn8C9ogElGI9HaTN8+YK9AjKhgdTDgE5q3Nt4gpGeohm6jDf+/ND3EGAhTnO4iwc9fCqhZdtYex8+G1KO4yQkN3pKx3LT0d2MIekmNFJeXbuoli0SRHPSMrQyJSdR2Ko7tlG49umTA2wK7jsSxDSjt8AEkYV0uMD4LK3KAAqNfcTEtO+DTjT2In/GzBB5U9DVdPRwNPRTklR5vp0FAKm4QIz0FfUTEClHZi60GouDaEw3QXGznildg67TXKbWh7GTjhvkCWYmcbThRGK67MfB/YxYBDftpyn96KX2l4R9/t+nUcDXCzSVbwsAB/x/t3Va8v/tXuX/HwUKfi2KZCMLAq+y2f/KKPBjfP+cBgss1RqmhvmOMUMXp5plQU0FLgueKGk84zG6WSOjRHQP3Vs/cdzKm7/JiX0EwR8uAkC61hOuctNs6IUhTwrqXAyYU+9WxotNdug8ICUwjLDQC2KfwtNIMNTPP9zrhA33JbJ3SdQc6lvJZ/0ZVtqfQc5Ju3eCo28iQ1DQ4lH0+DWaBEgNwjxTsWQeHXietoDRQxnQBTrBgC0yuZyHrTsLTMf7PtRzhmmatHlyydAadYTtlz4rMsP2epHOZRwElxyHKIZvixFlHwvxni8WJPQ3c+NAY4cA83VERa7PTttJN1WQIA6Y7+4kpuXoHbtfGxj9dw1SjnMWq5HL+QpEWWGvBse90+N4sRDoER1B9QsOKH8tJiAJn6iYAr67wR2vQ0/m1bRjNDYLOf7wiNqk0NksxT85nqVwkRIYZPjbI6/MLtPD5bN4h+TymdRbVLkZLBBMPm9SYZ2dfuQshPqLep4WDZd5M7KL4c354NX51c35m/Mzved4Mxq8PR9fDs7Os54ASz3Qb4Iv+rlGgCmjgX9Fp8XWpF17oH7mDd0siGV9BZU8Fh6VeXSdSG+l31k/lCpMku5WOy/Xkgfxgr7VXkKWBbS2mRtjoTta9h5q6LX8gCW/sjWSlxaAeQG/sv5LwadTEgfqLfeRSrfdrPLxvwLsy/8FFlHHuAhwIP/vtLb3f1onp3r/r8r/vz/o/Y18DWDmnMRqzgX7rz0WuP2nyYA2u0MB6oyKKx7Qr68M/pY5v4gD7TodvRv0WvA4MgI4kNv+Ke771LZihQOeVZ40L8WYvbOtgfyo2H6yIbfwvPm8pGKSjDGjyvwGTNqHlS43zFOUPcURzhsty1Kv72A6decy980UcfZ7OQCg3vS7KXr0ptZO/lbbzGw43M2WY0pU8zDJcO+12ASB5WYn+7AlYBbiLXG6xEhtHyX1BE2eaehHmJskbwWlOLDZLEkRTQlQeCG2HpB5S8C8h5YaJrjQWDiz7ZsepU8f+cQ+RNzfPDQCPrMUkSBfpLJiAGYhM+VccT7qz+tlhSd7hq4kkd3s3KU3jVkm9U0u5aWV72fzLCh5ko6m87BHcbVsDznnih+kJhlPPuLSNM7MkhoXStbvsdHzo0NdBTtgX/5X9FpfnwkeOv/tnm6d/7ZbrWa7yv8eA3ae/235gmr7t+ywf/S8HQv2rn97HGfO8r6lDjy4/lut7fOfXq9a/48C+fWfHr9uZ2dj236m7WCHM3jIUbAxNz0aFX19z/A2CuIZC11PMhctcEIw5THZpscXNRIEfPXObFid30UktFxiaURrERFIUSU75Bp7i/mpVOsImcN139UHHebFkL6R0tZlxL8Ig3UfpiSQ9P9mQT8Qyut/aafyiH8Acuj+T7tTuv/VOanOfx8F7BmRPcdMzoRw0cTuzBN6GWXxE+1EFwxZw76DHEVmfTAhRNcRUe7gaDgdcXWpr4tjWlHbFJPw6UutljsjSC7j6NfIXIqtJbvS+bOGPvSwubhfsqcjdr3vTCT1AZBsq9xLpVY+sNAVVJ7V3FFH2vs9ncw5vy113XqldzgDKMNWs73JYtSjD1BrpWs/+V18Qbk0zGd/U9Nutjtuz21aDfhppxvdftO9ad6cdG86zdc3xvtKeqPv/zc7rV/c5Vy7yc19oK3bQLm7QMU9L2dKzLybTtmNn3bvNTONxZs8m3s89SY8b7S78Fz/q/+s7riCCiqooIIKKqigggoqqKCCCiqooIIKKqigggqOAv8DOHt21ABQAAA= values: image: tag: 0.7.0-dev diff --git a/controllers/provider-alicloud/pkg/alicloud/secret.go b/controllers/provider-alicloud/pkg/alicloud/secret.go index 4001799c0..c5d5604bd 100644 --- a/controllers/provider-alicloud/pkg/alicloud/secret.go +++ b/controllers/provider-alicloud/pkg/alicloud/secret.go @@ -31,6 +31,8 @@ const ( AccessKeyID = "accessKeyID" // AccessKeySecret is the data field in a secret where the access key secret is stored at. AccessKeySecret = "accessKeySecret" + // StorageEndpoint is the data field in a secret where the storage endpoint is stored at. + StorageEndpoint = "storageEndpoint" ) // ReadSecretCredentials reads the Credentials from the given secret. diff --git a/controllers/provider-alicloud/pkg/alicloud/types.go b/controllers/provider-alicloud/pkg/alicloud/types.go index 19e36f432..14642d757 100644 --- a/controllers/provider-alicloud/pkg/alicloud/types.go +++ b/controllers/provider-alicloud/pkg/alicloud/types.go @@ -19,9 +19,13 @@ import "path/filepath" const ( // Name is the name of the Alicloud provider. Name = "provider-alicloud" + // StorageProviderName is the name of the Alicloud storage provider. + StorageProviderName = "OSS" // InfraRelease is the name of the alicloud-infra chart. InfraRelease = "alicloud-infra" + // ETCDBackupRestoreImageName is the name of the etcd backup and restore image. + ETCDBackupRestoreImageName = "etcd-backup-restore" // MachineControllerManagerImageName is the name of the MachineControllerManager image. MachineControllerManagerImageName = "machine-controller-manager" @@ -38,10 +42,17 @@ const ( // CSIPluginImageName is the name of the CSI plugin image. CSIPluginImageName = "csi-plugin-alicloud" + // BucketName is a constant for the key in a backup secret that holds the bucket name. + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + BucketName = "bucketName" + // CloudProviderConfigName is the name of the configmap containing the cloud provider config. CloudProviderConfigName = "cloud-provider-config" // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" + // BackupSecretName is the name of the secret containing the credentials for storing the backups of Shoot clusters. + BackupSecretName = "etcd-backup" ) var ( diff --git a/controllers/provider-alicloud/pkg/apis/config/types.go b/controllers/provider-alicloud/pkg/apis/config/types.go index 1f74923e9..143a5f20c 100644 --- a/controllers/provider-alicloud/pkg/apis/config/types.go +++ b/controllers/provider-alicloud/pkg/apis/config/types.go @@ -15,6 +15,7 @@ package config import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to Alicloud-specific identifiers. MachineImages []MachineImage + // ETCD is the etcd configuration. + ETCD ETCD } // MachineImage is a mapping from logical names and versions to Alicloud-specific identifiers. @@ -38,3 +41,25 @@ type MachineImage struct { // ID is the id of the image. ID string } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + ClassName *string + // Capacity is the storage capacity used in etcd-main volume claims. + Capacity *resource.Quantity +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + Schedule *string +} diff --git a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/types.go b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/types.go index d95157cd7..95237c216 100644 --- a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/types.go +++ b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to Alicloud-specific identifiers. MachineImages []MachineImage `json:"machineImages,omitempty"` + // ETCD is the etcd configuration. + ETCD ETCD `json:"etcd"` } // MachineImage is a mapping from logical names and versions to Alicloud-specific identifiers. @@ -38,3 +41,28 @@ type MachineImage struct { // ID is the id of the image. ID string `json:"id"` } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage `json:"storage"` + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup `json:"backup"` +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + // +optional + ClassName *string `json:"className,omitempty"` + // Capacity is the storage capacity used in etcd-main volume claims. + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty"` +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + // +optional + Schedule *string `json:"schedule,omitempty"` +} diff --git a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.conversion.go b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.conversion.go index 6ecaa52f8..e0bfd297e 100644 --- a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.conversion.go +++ b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.conversion.go @@ -24,6 +24,7 @@ import ( unsafe "unsafe" config "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + resource "k8s.io/apimachinery/pkg/api/resource" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -45,6 +46,36 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ETCD)(nil), (*config.ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCD_To_config_ETCD(a.(*ETCD), b.(*config.ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCD)(nil), (*ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCD_To_v1alpha1_ETCD(a.(*config.ETCD), b.(*ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDBackup)(nil), (*config.ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(a.(*ETCDBackup), b.(*config.ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDBackup)(nil), (*ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(a.(*config.ETCDBackup), b.(*ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDStorage)(nil), (*config.ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(a.(*ETCDStorage), b.(*config.ETCDStorage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDStorage)(nil), (*ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(a.(*config.ETCDStorage), b.(*ETCDStorage), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MachineImage)(nil), (*config.MachineImage)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_MachineImage_To_config_MachineImage(a.(*MachineImage), b.(*config.MachineImage), scope) }); err != nil { @@ -60,6 +91,9 @@ func RegisterConversions(s *runtime.Scheme) error { func autoConvert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration(in *ControllerConfiguration, out *config.ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]config.MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_v1alpha1_ETCD_To_config_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -70,6 +104,9 @@ func Convert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration( func autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in *config.ControllerConfiguration, out *ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_config_ETCD_To_v1alpha1_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -78,6 +115,78 @@ func Convert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration( return autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in, out, s) } +func autoConvert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + if err := Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_ETCD_To_config_ETCD is an autogenerated conversion function. +func Convert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCD_To_config_ETCD(in, out, s) +} + +func autoConvert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + if err := Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_config_ETCD_To_v1alpha1_ETCD is an autogenerated conversion function. +func Convert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + return autoConvert_config_ETCD_To_v1alpha1_ETCD(in, out, s) +} + +func autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup is an autogenerated conversion function. +func Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in, out, s) +} + +func autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup is an autogenerated conversion function. +func Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + return autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in, out, s) +} + +func autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage is an autogenerated conversion function. +func Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in, out, s) +} + +func autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage is an autogenerated conversion function. +func Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + return autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in, out, s) +} + func autoConvert_v1alpha1_MachineImage_To_config_MachineImage(in *MachineImage, out *config.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index e8748f09d..3b508b3d5 100644 --- a/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/controllers/provider-alicloud/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -33,6 +33,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { *out = make([]MachineImage, len(*in)) copy(*out, *in) } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -54,6 +55,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-alicloud/pkg/apis/config/zz_generated.deepcopy.go b/controllers/provider-alicloud/pkg/apis/config/zz_generated.deepcopy.go index d96f9c710..a0d0b5fca 100644 --- a/controllers/provider-alicloud/pkg/apis/config/zz_generated.deepcopy.go +++ b/controllers/provider-alicloud/pkg/apis/config/zz_generated.deepcopy.go @@ -33,6 +33,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { *out = make([]MachineImage, len(*in)) copy(*out, *in) } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -54,6 +55,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-alicloud/pkg/cmd/config.go b/controllers/provider-alicloud/pkg/cmd/config.go index 766a6ff6b..73cf51774 100644 --- a/controllers/provider-alicloud/pkg/cmd/config.go +++ b/controllers/provider-alicloud/pkg/cmd/config.go @@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) { *machineImages = c.Config.MachineImages } +// ApplyETCDStorage sets the given etcd storage configuration to that of this Config. +func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { + *etcdStorage = c.Config.ETCD.Storage +} + +// ApplyETCDBackup sets the given etcd backup configuration to that of this Config. +func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) { + *etcdBackup = c.Config.ETCD.Backup +} + // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/controllers/provider-alicloud/pkg/cmd/options.go b/controllers/provider-alicloud/pkg/cmd/options.go index 3830b069d..4a293bf11 100644 --- a/controllers/provider-alicloud/pkg/cmd/options.go +++ b/controllers/provider-alicloud/pkg/cmd/options.go @@ -18,10 +18,15 @@ import ( controlplanecontroller "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/controlplane" infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/controller/worker" + controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/webhook/controlplane" + controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/webhook/controlplanebackup" + controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/webhook/controlplaneexposure" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane" extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" extensionsworkercontroller "github.com/gardener/gardener-extensions/pkg/controller/worker" + webhookcmd "github.com/gardener/gardener-extensions/pkg/webhook/cmd" + extensioncontrolplanewebhook "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" ) // ControllerSwitchOptions are the controllercmd.SwitchOptions for the provider controllers. @@ -32,3 +37,12 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { controllercmd.Switch(extensionsworkercontroller.ControllerName, workercontroller.AddToManager), ) } + +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions( + webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager), + ) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplane/add.go b/controllers/provider-alicloud/pkg/webhook/controlplane/add.go new file mode 100644 index 000000000..437c86cf5 --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplane/add.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/alicloud" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = log.Log.WithName("alicloud-controlplane-webhook") + +// AddToManager creates a webhook and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.ShootKind, + Provider: alicloud.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &extensionsv1alpha1.OperatingSystemConfig{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(logger), controlplane.NewUnitSerializer(), + controlplane.NewKubeletConfigCodec(controlplane.NewFileContentInlineCodec()), logger), + }) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer.go b/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer.go new file mode 100644 index 000000000..79777b3ef --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer.go @@ -0,0 +1,110 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/coreos/go-systemd/unit" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// NewEnsurer creates a new controlplane ensurer. +func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + logger: logger.WithName("alicloud-controlplane-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + logger logr.Logger +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-apiserver"); c != nil { + ensureKubeAPIServerCommandLineArgs(c) + } + return nil +} + +// EnsureKubeControllerManagerDeployment ensures that the kube-controller-manager deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeControllerManagerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + ps := &dep.Spec.Template.Spec + if c := controlplane.ContainerWithName(ps.Containers, "kube-controller-manager"); c != nil { + ensureKubeControllerManagerCommandLineArgs(c) + } + return nil +} + +func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { + // Ensure CSI-related admission plugins + c.Command = controlplane.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") + + // Ensure CSI-related feature gates + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "VolumeSnapshotDataSource=true", ",") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "CSINodeInfo=true", ",") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "CSIDriverRegistry=true", ",") + c.Command = controlplane.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "KubeletPluginsWatcher=true", ",") +} + +func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") +} + +// EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. +func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, opts []*unit.UnitOption) ([]*unit.UnitOption, error) { + if opt := controlplane.UnitOptionWithSectionAndName(opts, "Service", "ExecStart"); opt != nil { + command := controlplane.DeserializeCommandLine(opt.Value) + command = ensureKubeletCommandLineArgs(command) + opt.Value = controlplane.SerializeCommandLine(command, 1, " \\\n ") + } + return opts, nil +} + +func ensureKubeletCommandLineArgs(command []string) []string { + command = controlplane.EnsureStringWithPrefix(command, "--provider-id=", "${PROVIDER_ID}") + command = controlplane.EnsureStringWithPrefix(command, "--cloud-provider=", "external") + command = controlplane.EnsureStringWithPrefix(command, "--enable-controller-attach-detach=", "true") + return command +} + +// EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. +func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { + // Ensure CSI-related feature gates + if kubeletConfig.FeatureGates == nil { + kubeletConfig.FeatureGates = make(map[string]bool) + } + kubeletConfig.FeatureGates["VolumeSnapshotDataSource"] = true + kubeletConfig.FeatureGates["CSINodeInfo"] = true + kubeletConfig.FeatureGates["CSIDriverRegistry"] = true + kubeletConfig.FeatureGates["KubeletPluginsWatcher"] = true + return nil +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer_test.go new file mode 100644 index 000000000..a9d9769c8 --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplane/ensurer_test.go @@ -0,0 +1,257 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/test" + + "github.com/coreos/go-systemd/unit" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Alicloud Controlplane Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + ctrl *gomock.Controller + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{ + "--enable-admission-plugins=Priority,PersistentVolumeLabel", + "--disable-admission-plugins=", + "--feature-gates=Foo=true", + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeAPIServerDeployment method and check the result + err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureKubeControllerManagerDeployment", func() { + It("should add missing elements to kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllerManagerDeploymentName}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + Command: []string{ + "--cloud-provider=?", + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err := ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + }) + + Describe("#EnsureKubeletServiceUnitOptions", func() { + It("should modify existing elements of kubelet.service unit options", func() { + var ( + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --provider-id=${PROVIDER_ID} \ + --cloud-provider=external \ + --enable-controller-attach-detach=true`, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletServiceUnitOptions method and check the result + opts, err := ensurer.EnsureKubeletServiceUnitOptions(context.TODO(), oldUnitOptions) + Expect(err).To(Not(HaveOccurred())) + Expect(opts).To(Equal(newUnitOptions)) + }) + }) + + Describe("#EnsureKubeletConfiguration", func() { + It("should modify existing elements of kubelet configuration", func() { + var ( + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + "CSIDriverRegistry": true, + "KubeletPluginsWatcher": true, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletConfiguration method and check the result + kubeletConfig := *oldKubeletConfig + err := ensurer.EnsureKubeletConfiguration(context.TODO(), &kubeletConfig) + Expect(err).To(Not(HaveOccurred())) + Expect(&kubeletConfig).To(Equal(newKubeletConfig)) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(Not(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ","))) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ",")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--feature-gates=", "VolumeSnapshotDataSource=true", ",")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--feature-gates=", "CSINodeInfo=true", ",")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--feature-gates=", "CSIDriverRegistry=true", ",")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--feature-gates=", "KubeletPluginsWatcher=true", ",")) +} + +func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { + // Check that the kube-controller-manager container still exists and contains all needed command line args, + // env vars, and volume mounts + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--cloud-provider=external")) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplanebackup/add.go b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/add.go new file mode 100644 index 000000000..8a409840b --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/alicloud" + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/imagevector" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the AWS backup webhook to the manager. +type AddOptions struct { + // ETCDBackup is the etcd backup configuration. + ETCDBackup config.ETCDBackup +} + +var logger = log.Log.WithName("alicloud-controlplanebackup-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.BackupKind, + Provider: alicloud.Type, + Types: []runtime.Object{&appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDBackup, imagevector.ImageVector(), logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer.go b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer.go new file mode 100644 index 000000000..332c68f9f --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer.go @@ -0,0 +1,136 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/alicloud" + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdBackup *config.ETCDBackup, imageVector imagevector.ImageVector, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdBackup: etcdBackup, + imageVector: imageVector, + logger: logger.WithName("alicloud-controlplanebackup-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdBackup *config.ETCDBackup + imageVector imagevector.ImageVector + logger logr.Logger +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + if err := e.ensureContainers(&ss.Spec.Template.Spec, ss.Name, cluster); err != nil { + return err + } + return nil +} + +func (e *ensurer) ensureContainers(ps *corev1.PodSpec, name string, cluster *extensionscontroller.Cluster) error { + c, err := e.getBackupRestoreContainer(name, cluster) + if err != nil { + return err + } + ps.Containers = controlplane.EnsureContainerWithName(ps.Containers, *c) + return nil +} + +func (e *ensurer) getBackupRestoreContainer(name string, cluster *extensionscontroller.Cluster) (*corev1.Container, error) { + // Find etcd-backup-restore image + // TODO Get seed version from clientset when it's possible to inject it + image, err := e.imageVector.FindImage(alicloud.ETCDBackupRestoreImageName, "", cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, errors.Wrapf(err, "could not find image %s", alicloud.ETCDBackupRestoreImageName) + } + + const ( + defaultSchedule = "0 */24 * * *" + ) + + // Determine provider and container env variables + // They are only specified for the etcd-main stateful set (backup is enabled) + var ( + provider string + env []corev1.EnvVar + ) + if name == common.EtcdMainStatefulSetName { + provider = alicloud.StorageProviderName + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ENDPOINT", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.StorageEndpoint, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.AccessKeyID, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ACCESS_KEY_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.AccessKeySecret, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + } + } + + // Determine schedule + var schedule = defaultSchedule + if e.etcdBackup.Schedule != nil { + schedule = defaultSchedule + } + + return controlplane.GetBackupRestoreContainer(name, schedule, provider, image.String(), nil, env, nil), nil +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer_test.go b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer_test.go new file mode 100644 index 000000000..7ae2143f7 --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplanebackup/ensurer_test.go @@ -0,0 +1,207 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/alicloud" + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Alicloud Controlplane Backup Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + Describe("#EnsureETCDStatefulSet", func() { + var ( + etcdBackup = &config.ETCDBackup{ + Schedule: util.StringPtr("0 */24 * * *"), + } + + imageVector = imagevector.ImageVector{ + { + Name: alicloud.ETCDBackupRestoreImageName, + Repository: "test-repository", + Tag: "test-tag", + }, + } + + cluster = &extensionscontroller.Cluster{ + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + ) + + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + var ( + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ENDPOINT", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.StorageEndpoint, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.AccessKeyID, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + { + Name: "ALICLOUD_ACCESS_KEY_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: alicloud.AccessKeySecret, + LocalObjectReference: corev1.LocalObjectReference{Name: alicloud.BackupSecretName}, + }, + }, + }, + } + ) + + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdMainStatefulSetName, "0 */24 * * *", alicloud.StorageProviderName, + "test-repository:test-tag", nil, env, nil))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdEventsStatefulSetName, "0 */24 * * *", "", + "test-repository:test-tag", nil, nil, nil))) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/add.go new file mode 100644 index 000000000..b204edd4d --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/add.go @@ -0,0 +1,58 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/alicloud" + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the AWS exposure webhook to the manager. +type AddOptions struct { + // ETCDStorage is the etcd storage configuration. + ETCDStorage config.ETCDStorage +} + +var logger = log.Log.WithName("alicloud-controlplaneexposure-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.SeedKind, + Provider: alicloud.Type, + Types: []runtime.Object{&appsv1.Deployment{}, &appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDStorage, logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer.go b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer.go new file mode 100644 index 000000000..b583974a9 --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer.go @@ -0,0 +1,86 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdStorage *config.ETCDStorage, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdStorage: etcdStorage, + logger: logger.WithName("alicloud-controlplaneexposure-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdStorage *config.ETCDStorage + client client.Client + logger logr.Logger +} + +// InjectClient injects the given client into the ensurer. +func (m *ensurer) InjectClient(client client.Client) error { + m.client = client + return nil +} + +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1.Deployment) error { + // Get load balancer address of the kube-apiserver service + address, err := controlplane.GetLoadBalancerIngress(ctx, e.client, dep.Namespace, common.KubeAPIServerDeploymentName) + if err != nil { + return errors.Wrap(err, "could not get kube-apiserver service load balancer address") + } + + if c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver"); c != nil { + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--advertise-address=", address) + c.Command = controlplane.EnsureStringWithPrefix(c.Command, "--external-hostname=", address) + } + return nil +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + e.ensureVolumeClaimTemplates(&ss.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureVolumeClaimTemplates(spec *appsv1.StatefulSetSpec, name string) { + t := e.getVolumeClaimTemplate(name) + spec.VolumeClaimTemplates = controlplane.EnsurePVCWithName(spec.VolumeClaimTemplates, *t) +} + +func (e *ensurer) getVolumeClaimTemplate(name string) *corev1.PersistentVolumeClaim { + var etcdStorage config.ETCDStorage + if name == common.EtcdMainStatefulSetName { + etcdStorage = *e.etcdStorage + } + return controlplane.GetETCDVolumeClaimTemplate(name, etcdStorage.ClassName, etcdStorage.Capacity) +} diff --git a/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer_test.go b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer_test.go new file mode 100644 index 000000000..1635a9e39 --- /dev/null +++ b/controllers/provider-alicloud/pkg/webhook/controlplaneexposure/ensurer_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplaneexposure + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-alicloud/pkg/apis/config" + mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Alicloud Controlplane Exposure Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + etcdStorage = &config.ETCDStorage{ + ClassName: util.StringPtr("gardener.cloud-fast"), + Capacity: util.QuantityPtr(resource.MustParse("25Gi")), + } + + ctrl *gomock.Controller + + svcKey = client.ObjectKey{Namespace: namespace, Name: common.KubeAPIServerDeploymentName} + svc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "1.2.3.4"}, + }, + }, + }, + } + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: common.KubeAPIServerDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{"--advertise-address=?", "--external-hostname=?"}, + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep) + }) + }) + + Describe("#EnsureETCDStatefulSet", func() { + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-main"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-events"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { + // Check that the kube-apiserver container still exists and contains all needed command line args + c := controlplane.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(ContainElement("--advertise-address=1.2.3.4")) + Expect(c.Command).To(ContainElement("--external-hostname=1.2.3.4")) +} + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-main") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdMainStatefulSetName, util.StringPtr("gardener.cloud-fast"), + util.QuantityPtr(resource.MustParse("25Gi"))))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-events") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdEventsStatefulSetName, nil, nil))) +} + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *corev1.Service: + *obj.(*corev1.Service) = *result.(*corev1.Service) + } + return nil + } +}