From 7ce38d08e0a0dc8767a692c5559767f7dfb3a0ef Mon Sep 17 00:00:00 2001 From: Stoyan Rachev Date: Mon, 27 May 2019 15:59:22 +0300 Subject: [PATCH] Implement customizing of etcd stateful sets via webhooks for AWS and GCP --- Gopkg.lock | 1 + controllers/provider-aws/charts/images.yaml | 4 + .../provider-aws/templates/configmap.yaml | 6 + .../provider-aws/templates/storageclass.yaml | 8 + .../charts/provider-aws/values.yaml | 7 + .../app/app.go | 32 +-- .../example/00-componentconfig.yaml | 6 + .../example/30-etcd-backup-secret.yaml | 11 + .../example/controller-registration.yaml | 2 +- .../provider-aws/pkg/apis/config/types.go | 25 +++ .../pkg/apis/config/v1alpha1/types.go | 28 +++ .../v1alpha1/zz_generated.conversion.go | 109 +++++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 66 ++++++ .../pkg/apis/config/zz_generated.deepcopy.go | 66 ++++++ controllers/provider-aws/pkg/aws/types.go | 10 + controllers/provider-aws/pkg/cmd/config.go | 10 + controllers/provider-aws/pkg/cmd/options.go | 15 +- .../pkg/webhook/controlplane/add.go | 5 +- .../pkg/webhook/controlplane/ensurer.go | 2 +- .../pkg/webhook/controlplane/ensurer_test.go | 2 +- .../pkg/webhook/controlplanebackup/add.go | 59 +++++ .../pkg/webhook/controlplanebackup/ensurer.go | 136 ++++++++++++ .../controlplanebackup/ensurer_test.go | 207 ++++++++++++++++++ .../pkg/webhook/controlplaneexposure/add.go | 25 ++- .../webhook/controlplaneexposure/ensurer.go | 32 ++- .../controlplaneexposure/ensurer_test.go | 124 ++++++++++- controllers/provider-gcp/charts/images.yaml | 4 + .../provider-gcp/templates/configmap.yaml | 6 + .../provider-gcp/templates/storageclass.yaml | 8 + .../charts/provider-gcp/values.yaml | 7 + .../app/app.go | 52 +++-- .../example/00-componentconfig.yaml | 6 + .../example/30-etcd-backup-secret.yaml | 9 + .../example/controller-registration.yaml | 2 +- .../provider-gcp/pkg/apis/config/types.go | 25 +++ .../pkg/apis/config/v1alpha1/types.go | 28 +++ .../v1alpha1/zz_generated.conversion.go | 109 +++++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 66 ++++++ .../pkg/apis/config/zz_generated.deepcopy.go | 66 ++++++ controllers/provider-gcp/pkg/cmd/config.go | 10 + controllers/provider-gcp/pkg/cmd/options.go | 13 +- controllers/provider-gcp/pkg/gcp/types.go | 11 + .../pkg/webhook/controlplane/add.go | 2 +- .../pkg/webhook/controlplane/ensurer.go | 2 +- .../pkg/webhook/controlplane/ensurer_test.go | 3 +- .../pkg/webhook/controlplanebackup/add.go | 59 +++++ .../pkg/webhook/controlplanebackup/ensurer.go | 137 ++++++++++++ .../controlplanebackup/ensurer_test.go | 202 +++++++++++++++++ .../pkg/webhook/controlplaneexposure/add.go | 27 ++- .../webhook/controlplaneexposure/ensurer.go | 34 ++- .../controlplaneexposure/ensurer_test.go | 121 +++++++++- .../app/app.go | 28 +-- .../provider-packet/pkg/cmd/options.go | 6 +- .../controlplane/genericmutator/mocks.go | 15 ++ pkg/util/pointers.go | 7 + pkg/webhook/cmd/options.go | 7 +- pkg/webhook/controlplane/controlplane.go | 10 +- pkg/webhook/controlplane/etcd.go | 128 +++++++++++ .../controlplane/genericmutator/mutator.go | 16 ++ .../genericmutator/mutator_test.go | 120 ++++++++++ .../{ => genericmutator}/noopensurer.go | 9 +- pkg/webhook/controlplane/utils.go | 55 +++++ pkg/webhook/utils.go | 10 +- 63 files changed, 2309 insertions(+), 109 deletions(-) create mode 100644 controllers/provider-aws/charts/provider-aws/templates/storageclass.yaml create mode 100644 controllers/provider-aws/example/30-etcd-backup-secret.yaml create mode 100644 controllers/provider-aws/pkg/webhook/controlplanebackup/add.go create mode 100644 controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer.go create mode 100644 controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer_test.go create mode 100644 controllers/provider-gcp/charts/provider-gcp/templates/storageclass.yaml create mode 100644 controllers/provider-gcp/example/30-etcd-backup-secret.yaml create mode 100644 controllers/provider-gcp/pkg/webhook/controlplanebackup/add.go create mode 100644 controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer.go create mode 100644 controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer_test.go create mode 100644 pkg/webhook/controlplane/etcd.go rename pkg/webhook/controlplane/{ => genericmutator}/noopensurer.go (88%) diff --git a/Gopkg.lock b/Gopkg.lock index 362183317..324f9e9b1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1839,6 +1839,7 @@ "k8s.io/apimachinery/pkg/api/equality", "k8s.io/apimachinery/pkg/api/errors", "k8s.io/apimachinery/pkg/api/meta", + "k8s.io/apimachinery/pkg/api/resource", "k8s.io/apimachinery/pkg/apis/config", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", diff --git a/controllers/provider-aws/charts/images.yaml b/controllers/provider-aws/charts/images.yaml index a53859762..987b12017 100644 --- a/controllers/provider-aws/charts/images.yaml +++ b/controllers/provider-aws/charts/images.yaml @@ -10,3 +10,7 @@ images: sourceRepository: github.com/gardener/machine-controller-manager repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager tag: "0.18.0" +- name: etcd-backup-restore + sourceRepository: github.com/gardener/etcd-backup-restore + repository: eu.gcr.io/gardener-project/gardener/etcdbrctl + tag: "0.6.3" diff --git a/controllers/provider-aws/charts/provider-aws/templates/configmap.yaml b/controllers/provider-aws/charts/provider-aws/templates/configmap.yaml index ad5320d99..d30aa2065 100644 --- a/controllers/provider-aws/charts/provider-aws/templates/configmap.yaml +++ b/controllers/provider-aws/charts/provider-aws/templates/configmap.yaml @@ -15,3 +15,9 @@ data: machineImages: {{ toYaml .Values.config.machineImages | indent 4 }} {{- end }} + etcd: + storage: + className: {{ .Values.config.etcd.storage.className }} + capacity: {{ .Values.config.etcd.storage.capacity }} + backup: + schedule: {{ .Values.config.etcd.backup.schedule }} diff --git a/controllers/provider-aws/charts/provider-aws/templates/storageclass.yaml b/controllers/provider-aws/charts/provider-aws/templates/storageclass.yaml new file mode 100644 index 000000000..77dddf6cf --- /dev/null +++ b/controllers/provider-aws/charts/provider-aws/templates/storageclass.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.config.etcd.storage.className }} +provisioner: kubernetes.io/aws-ebs +allowVolumeExpansion: true +parameters: + type: gp2 \ No newline at end of file diff --git a/controllers/provider-aws/charts/provider-aws/values.yaml b/controllers/provider-aws/charts/provider-aws/values.yaml index 6b2026718..d05e8980e 100644 --- a/controllers/provider-aws/charts/provider-aws/values.yaml +++ b/controllers/provider-aws/charts/provider-aws/values.yaml @@ -19,6 +19,7 @@ disableControllers: disableWebhooks: - controlplane - controlplaneexposure +- controlplanebackup config: machineImages: @@ -37,3 +38,9 @@ config: ami: ami-07739b17529e8c1d0 - name: ap-southeast-2 ami: ami-02d7d488d701a460e + etcd: + storage: + className: gardener.cloud-fast + capacity: 80Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-aws/cmd/gardener-extension-provider-aws/app/app.go b/controllers/provider-aws/cmd/gardener-extension-provider-aws/app/app.go index 34eb89dc6..039079a2a 100644 --- a/controllers/provider-aws/cmd/gardener-extension-provider-aws/app/app.go +++ b/controllers/provider-aws/cmd/gardener-extension-provider-aws/app/app.go @@ -25,6 +25,8 @@ import ( awscontrolplane "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/controller/controlplane" awsinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/controller/infrastructure" awsworker "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/controller/worker" + awscontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplanebackup" + awscontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplaneexposure" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" @@ -64,8 +66,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { MaxConcurrentReconciles: 5, } - controllerSwitches = awscmd.ControllerSwitchOptions() - webhookSwitches = awscmd.WebhookAddToManagerOptions() + controllerSwitches = awscmd.ControllerSwitchOptions() + webhookSwitches = awscmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("aws-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, @@ -75,20 +87,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.PrefixOption("worker-", workerCtrlOpts), configFileOpts, controllerSwitches, - webhookSwitches, + webhookOptions, ) ) - webhookSwitches.Server = webhookcmd.ServerOptions{ - Port: 7890, - CertDir: "/tmp/cert", - Mode: webhookcmd.ServiceMode, - Name: "webhooks", - Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), - ServiceSelectors: "{}", - Host: "localhost", - } - cmd := &cobra.Command{ Use: fmt.Sprintf("%s-controller-manager", aws.Name), @@ -111,6 +113,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts.Completed().ApplyMachineImages(&awsworker.DefaultAddOptions.MachineImagesToAMIMapping) + configFileOpts.Completed().ApplyETCDStorage(&awscontrolplaneexposure.DefaultAddOptions.ETCDStorage) + configFileOpts.Completed().ApplyETCDBackup(&awscontrolplanebackup.DefaultAddOptions.ETCDBackup) controlPlaneCtrlOpts.Completed().Apply(&awscontrolplane.Options) infraCtrlOpts.Completed().Apply(&awsinfrastructure.DefaultAddOptions.Controller) infraReconcileOpts.Completed().Apply(&awsinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation) @@ -120,7 +124,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } - if err := webhookSwitches.Completed().AddToManager(mgr); err != nil { + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") } diff --git a/controllers/provider-aws/example/00-componentconfig.yaml b/controllers/provider-aws/example/00-componentconfig.yaml index 46ab2523c..34804d25f 100644 --- a/controllers/provider-aws/example/00-componentconfig.yaml +++ b/controllers/provider-aws/example/00-componentconfig.yaml @@ -17,3 +17,9 @@ machineImages: ami: ami-07739b17529e8c1d0 - name: ap-southeast-2 ami: ami-02d7d488d701a460e +etcd: + storage: + className: gardener.cloud-fast + capacity: 80Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-aws/example/30-etcd-backup-secret.yaml b/controllers/provider-aws/example/30-etcd-backup-secret.yaml new file mode 100644 index 000000000..6920bf017 --- /dev/null +++ b/controllers/provider-aws/example/30-etcd-backup-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-backup + namespace: shoot--foo--bar +type: Opaque +data: + bucketName: YnVja2V0MTIz # bucket123 + region: ZXUtd2VzdC0x # eu-west-1 + accessKeyID: YWRtaW4= # admin + secretAccessKey: YWRtaW4= # admin diff --git a/controllers/provider-aws/example/controller-registration.yaml b/controllers/provider-aws/example/controller-registration.yaml index 3636e4706..52a56002b 100644 --- a/controllers/provider-aws/example/controller-registration.yaml +++ b/controllers/provider-aws/example/controller-registration.yaml @@ -14,7 +14,7 @@ spec: deployment: type: helm providerConfig: - chart: H4sIAAAAAAAAA+0a227buDLP/oqBcR7ag0q+x6kP+uCm2V3jtGkQZ7fo04KWaJuNJGpJyq5P238/Q1KSJdnOZZtm0VaDIKJIzoXD4Vwox4KvmE+FQ9aydfRtoI0wHAzME6H6NO1Or9/pDrrHx7q/0+0MB0cw+EbylCCRigiAI8G5umnebePfKcTF/T9dEqHcDQmDB+Vx2/53O4PK/mO7dwTtB5XiAPzk+09i9gcVkvFoBKtOg8Rx/tp2h27b8emq4VPpCRYr0z2G32gQgqdtBeZcgFpS+JUIn0ZUwPjdFC5SmwL6UdFIE2tEJKQjKBpbY7XL559Wxk8IpfPvc89d8Afnccv57+Jo5fz3usP6/D8KtFpwyuONYIulgifeU+i2O89hOr6A6Rng4SaReSHzOQsYURQ8HsYk2rgwDgIwaBIElVSsqO/C1ZJJwKkU8BkwD48/9SGJtDfQfmIcEw8fUz5XayIovLZTnsHKhS76C4/GCoiEiCvE44gi1kwitcigv56cnp2jYJpDo9XCv4zCHiY57dSjQddtwxM9oZkONZ/+R5PY8ARCstFMIUFmKl9EKhBy18tGBUQehTVTSyuNpeJqGu9TGnymCE4niBDj27w4EYhKhTawVCoetVrr9dolRmKXi0UrVZpspWt1UOoU6/cooFJr+6+ECVzxbAPorxGBzFDWgKzNhi0ExTHFtdRrwRSLFs9ApgrXZHwmlWCzRJWUlsmISy9OQLWhCTTHU5hMm/ByPJ1Mn2ki7yZXv739/QrejS8vx+dXk7MpvL2E07fnryZXk7fn+PYLjM/fw38n56+eAWV6J1GdsdArQDGZVidajKY1pbQkQhZUZEw9NmceLi1aJGRBYcExakS4IoipCJnU2ypRQF+TCVjIFFGma2ddbgOnLPhooaOUtmPXbeV/S+Jdt7IRx+OREjwI0CkKutC6MERduSwFMHBTGvQjwcXQ1iE8nU/BJJoLgl2JpxJBRxr/1M6/wNXZjndcXFOhm1pYuECp9KJtpKWR3mUJxTXIJI55GoXTTq0bvWyPC0E9BVuhoCRUIy5SryPvzwql+K8oGjIa1gNXgvev//odHf/r+u/bw4H9R7cxZ4uQxA9RDd62/8Nhtf4btnvHdf73GFCp/65Z5I90ZMLdf0PiRkgV8YkiowaAreEWaaXn5MWdU7QhJ7ecFENioEG0T5/AvaQBJRiKz7Nu+PIFZwVkRgOpOYDOZ9zrZIZBnqIZuoy37sT1AC6LcHOjfew152xdVmJj6CP4bEg5jpOS3GoH2bgZTzcXQ7opeiah6wU88VurDgniJekYMrlW01Bs9ZvYUNz49MkBNgf3DxIkNKcXYkbIIjoJMUhLqyiAUucIMTHPe6/TixuRP2MqiLIp6Gs6mh2NfN08Onj+fRoHfBMi0gM4gFvO/2Cwe/67g/r8PwqULDyOZSt3Aq9yE7i/F3j0s7+kQYgJestks9/GXehqRAsqqCm5pJ2Vnru085QneMzMyiSie4qLUXpwlbd8XVjs1y33/tIDZIc7FaiwpYZehNWvLZ6yLvSMS+pdyyTcpgPO3dy/kYFFXpD4FJ7EgqFW/uVepRK4L1GyC4JVdLOSaDSfYlX1GeSSoANAxpnfKxvK12vv7+gPILMA06ZihZXl2PP0lp/fg7euyAg6Z5GvxrnzobLAtFsfQbNgf6ZLWyGXDI1ugyKPdoYVWWB/s0znIgmCC47GuykZtMWI88HtTugVhCHW3NvNcKC1R/blBov0wpyqnWTFMtJCXsWZTmpGjr6EedGiyttHvxS6LEKrEMur9DS7WNfayPCj7vASLJEjhcW6fkFO8kVBAVsB87bBdreY003kyaJiNCdWKvTvz6uMf09ubBFxfPCY2tzC2R7rO/KzFN5mBMY5fpXz2lxW3H99Fu+2da3pbMn5dWYHIffpi/TI3TRPn6MX2rzSfnnbZBOKXtwQoA5ip9I4mZ/HBTf3O6XmqHmL9Taf7UHN/BGi73FIzbJwPpP6eqhwrEobkA5vM1CdFH7gLAJkXV1oRivT4R5C79KhA1RotCo6B+vdXp+NX51d/nn2+uxU3xD+eT5+cza9GJ+e5TMBVprLL4KHo0InwJzRwL+k83Jv2q8DySiPZ26+q/lcfUUmd+XJ9tNcXYsC5dxBX3CdSwxPnrcLo7hvinscS4Wr04u8X1DJE+FRWRRRZ9qV/Dyfh5qL0qy80y3qbsWDJKRvdFTZI7Q1vgKPUE+0KriHn9ylS4U6QFWFcaswbOXbiV0Vul5WQRb1cf8CMgOfzkkSqDfoBkbQ77arzIvSS+oJqop8bY+N0Xudwx4GugLcWxtPDbE9KXGF8l3y3386+T86WP+JGfEe7IcAt9R/ve6gX6n/BvpKqK7/HgGqdm42niRqyQX7n/1UcH1i4tH2dihAnVFxyQP6tyrD76jmE0mgXZ2jL4J+FTyJjdgOFK5/yvc+jUoocMCz2pLmpZxs7e1roTwqsUPFvHNPT3GqTatK7e0wxrdZKs6CKvMMmLSNta5MTSvOW0mMe0p3l91s7llf5q5lYcx4OTte8u2oXd00KZO++tor2roqx1a4/RI55ubCNGY57kFDtgh++vWw9FGsOCFmhU3OByqLz5285U5XGM9t08Ycuasic/kWY9KUDm5v2jJEk1eWXoitMmXRoNAK6E7HDA8oixa2fztjZ+gDn9lGzP1toxXwhXkJE2W+JKbhzCteV6Y8kSUPM21g9GQRy0YLW9r8d3N3z9LLSVeS2JyZvZrVmLukvspZvbQa+OF9Fi41zZUzxd+gqUZ+O13w6nfVi0xmH/AkGw+ZZUfFa5F7KPg7SZd+ODiQ/5Udz1dmgrd9/+sPOuX8T/8ibFjnf48BB2qc0imur/9/XOdTOv8re0nx0D8Av+X8d/rDTvX3H73j+vvfo4D9lmDyr+zbwQho4i48oc9EfpLQTnSozztuuvBXZDECE0d0BhAXPjBM5udcXeifi6JbaWxTPvj0pdEo3F2mn+XzUsd6iMqd8QgG2F0un26YiFMP3W2PYE4CqfMeWzodpNLYvUjVuU/p68J2GY3KbWl1auWVfsQdwDUYTeiPaI2dD/7F+zZBufVk+Q/pO8+Ph+7AtbdjurDJP+ZlWDRxPFyPIIHTSW+/SMhG+p/TnvePvS71+8f+SWcwJKSKucb93IN23D2h/ZNerzOYDfzOkJbQEumgF92H5h0PEWVwQgceGfbmx3vRujtoA3/ea3tdfMxJm5J5CY3EDlqUWh5gORz2ns86w0H3OT3xOn77MO4u364/9PsnJ/6w3SH94zb9UeNBDTXUUEMNNdRQQw011FBDDTXUUEMNNdRQQw011FBDDd83/B+/n2MYAFAAAA== + chart: H4sIAAAAAAAAA+0bXXPTuJbn/IozmfsADHa+m5I7PITSZTsXSqfpwvC0o9hKImpbvpKc0Av893sk2Y7tJE27LWUAn2U2tqTzoSPpfMmNBV8ynwqHrGTr0feBNsJwMDC/CNVf89zp9TvdQffgQLd3up3h4BEMvpM8JUikIgLgkeBcXTduX/9PCnFx/Y8WRCj3ioTBvfLYt/7dzqCy/vjcewTte5ViB/zm609i9p4KyXg0gmWnQeI4f227Q7ft+HTZ8Kn0BIuVaR7DnzQIwdN7BWZcgFpQeE2ETyMqYPxhAmfpngL6WdFIE2tEJKQjKG62xnKTz49Wxm8IpfPvc8+d83vnsef8d7G3cv573WF9/h8EWi044vGVYPOFgsfeE+i2O89hMj6DyTHg4SaReSGzGQsYURQ8HsYkunJhHARg0CQIKqlYUt+FiwWTgEMp4G/APDz+1Ick0tZA24lxTDz8mfCZWhFB4Y0d8gyWLnTRXng0VkAkRFwhHkcUsWISqUUG/c3J0fEpCqY5NFot/JdR2MIkp51aNOi6bXisBzTTruaTf2sSVzyBkFxpppAgM5VPIhUIuetpowIij8KKqYWVxlJxNY2PKQ0+VQSHE0SI8W1WHAhEpUIbWCgVj1qt1WrlEiOxy8W8lSpNttK5Oih1ivVXFFCptf3fhAmc8fQK0F4jApmirAFZmQWbC4p9imupV4IpFs2fgUwVrsn4TCrBpokqKS2TEadeHIBqwy3QHE/gZNKEl+PJyeSZJvLh5OLPd39dwIfx+fn49OLkeALvzuHo3emrk4uTd6f49geMTz/Cf05OXz0DyvRKojpjoWeAYjKtTtwxmtaE0pIImVORMfXYjHk4tWiekDmFOUevEeGMIKYiZFIvq0QBfU0mYCFTRJmmjXm5DRwy56O59lJ6H7tuK/+3IN5lK+txPB4pwYMAjaKgc60LQ9SVi5IDAzelQT8TnAxt7cLT8RScRDNBsCnxVCLoSOMf2fFnODvb8IGLSyr0oxYWzlAqPWnraWmkV1lCcQ4yiWOeeuG0UetGT9vjQlBPwVooKAnViIvUa8/7u0LJ/yuKGxk31j1ngrfP//od7f/r/O/7w471R7MxY/OQxPeRDe5Z/06v16+s/7DdPajjv4eASv53ySJ/pD0Trv5bEjdCqohPFBk1AGwON08zPSdP7pziHnLynZNiSHQ0iPblC7jnNKAEXfFp1gzfvuGogExpIDUH0PGMe5lM0clT3IYu460bcd2ByyJc3Ggbe805m5eV2Gz0EXw1pBzHSUmutYNs3Iynm4sh3RQ9k9D1Ap74rWWHBPGCdAyZXKupK7b6Tawrbnz54gCbgfueBAnN6YUYEbKInoTopKVVFECpcYSYGOd91OHFtchfMRRE2RT0NR3NjkZ+RpIqz7e6B5CKC8TIXlE1AZHy1KyA1mCZicZ0UxQ3H5nRNeg6rmXqaj92OnCNPMXwJInXkkiMj/0k2C2IRXCzcZrSjz5bPwPssP8+jQN+FeKmuQcHsMf+DwbDav1v2B3U9v9BoGTh4li2cifwKt8Ct/cCD277FzQIMUFrmWzm+7gLnY1qQQU1KbcsWaK08YgnaGbNzCSie2jeRqnhVt7iTWGyd5vu7aUHyA53KlBhSQ29KOJp8lww/wvqXcokXIeDzs3cv5GBRV6Q+BQex4KhVv7lXqQSuC9RsjOiFtCsBJrNJ5hVfwW5IGgAkPHaHwQl3d1Ve/9EfwDZDjDPVCyZR8eep5f89Ba8dUZO0DmLfDbOjQ+VBabd+giahf1nmvQu5JLhptOOdLTRrcgc25tlOmdJEJxx3LxlL20x4ryz5NZ5GJLIXy+GA60tsi+uYioKY6r7JCuWIC3kVRzppNvI0UW4Fy3079vol1yXRWgVYrkqPc0u1rUWZPhZN3iJEGjdHEH1C3KSL8qxRSpg/myw3TXm5CryZFExmhMrFXpuz6uMf0tubB5x/OExtbGlsz7WN+RnKbzLCIxz/CrnlSlW3X5+Fm/fvFZ0uuD8MtsHIffpi/TIXTdOn6MXenul7XLfYOOKXlzjoHZip9I4mZ3HCTe3G6XmqLln9zafbUHN7BGibzFIzbJwPpO6PFg4VqUFSLvXGYhOCj5xFgGyrk40o5XpcAuhD2nXDio0WhaNg7Vub47Hr47P/z5+c3ykK8R/n47fHk/OxkfH+UiApebyh+DhqNAIMGM08M/prNyatmtHMsr9mZuvaj5Wl0jlpjzZepqrC1GgnBvoM65jieHh83ahF9dNcY9jqnhxdJa3Cyp5IjwqiyLqTKuSn+XjUHNRmpV1ukXdLXmQhPSt9ipbhLabr8Aj1AOtCm5hJzfpUqF2UFVh3Cp0W/k2fFeFrpdVEIr6uH0BIQOfzkgSqLdoBkbQ77arzIvSS+oJqop8bYv10VuNwxYGugKwtTYyMcS2hMQVyjeJf3908P9oZ/4nMJ2+tw9B9uR/ve6gWv8bDPX3P3X+9/2hus/NwpNELbhg/7NXRZeHxh+tq4MB6oyKcx7Qf5QZ/kQ5n0gCbeocXQh8LXgSG7EdKJT/ynW/RsUVOOBZbUnzUg62tra1UB6V2K5i3LmlpTjUhlWl53U3+rdpKs6cKvMbMGkfVjozNU9x/pTEuKZ0c9rN5pb5ZeZaFvqMlbP9JduO2tWPJmTSpc+toq2qcqyF2y6RYyoX5mGa4+7cyBbBT2+PS5eixQExKyxy3lGZfG7kLXe6RH9uH63PkZsqMsXXGIOmtHNdacsQTVxZeiE2y5TFDYW7gG40TPGAsmhu29cjNro+8al9iLm/fmgFfG5ewkSZm+TUnXnFcnXKE1nyMNMGek8Wsay3sKTNp83NNUuL064ksTkzWzWrMTdJ3clYvbQa+OVtFk41jZUzxV+jqUZ+O1Gw6jfVi0ymn/AkGwuZRUfFssgtFPyThEu/HOyI/8qG546R4L77//6gU47/9BeBwzr+ewjYkeOUTnFd/v91jc+u828vZs2t7p3zwH3nv9fe+P5jgCahPv8PAMXzn93GV2OoiW0/0pthizG4zZcBZrtpblSMoHz8dH5AMSwlQcBX702V6fhzTCIrG+ZHtBETgXRUenWirmJtHuLuj9bhzwyl87+0q3jffwC07/uvYWfj+7/eQbc+/w8B9i7R5F/Z3eEIaOLOPaEPZe5JcZ/oUD9vuO7CT5H5CIwf0RlAXLhgPJmdcnWm/1wAw4rGOuWDL98ajcLdRfpZVl7qsBFC5c5oBANsLpdPrhmIQ3fdbY1gRgKp8x5bOtlJpbF5kaJzn9Lt4noajcptSXVo5ZV+xhXAOVSa7YdNRj36Zr2x8RVYsQgvKLfhTf7XVZ3nB0N34NqSua525Df8GRZNHA8nKUjgdNKSOAnZSP/Pac/6B16X+v0D/7AzGBJSxVzhIm9BO+ge0v5hr9cZTAd+Z0hLaAkaerIVzTsYIsrgkA48MuzNDraidTfQBv6s1/a6+DMjbUpmJTQSO7jN1GIHy+Gw93yKAUf3OT30On57N+4m364/9PuHh/6w3SH9g7ae5fpzusrHdIVP6colQ2dGZHZ7sf5g7rD9mpnG8odw68/gmm142ur24an+r/mrhqc11FBDDTXUUEMNNdRQQw011FBDDTXUUMMd4P84QBVdAFAAAA== values: image: tag: 0.7.0-dev diff --git a/controllers/provider-aws/pkg/apis/config/types.go b/controllers/provider-aws/pkg/apis/config/types.go index d3c6f810b..c8d8bd91e 100644 --- a/controllers/provider-aws/pkg/apis/config/types.go +++ b/controllers/provider-aws/pkg/apis/config/types.go @@ -15,6 +15,7 @@ package config import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to AWS-specific identifiers, i.e. AMIs. MachineImages []MachineImage + // ETCD is the etcd configuration. + ETCD ETCD } // MachineImage is a mapping from logical names and versions to AWS-specific identifiers, i.e. AMIs. @@ -46,3 +49,25 @@ type RegionAMIMapping struct { // AMI is the AMI for the machine image. AMI string } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + ClassName *string + // Capacity is the storage capacity used in etcd-main volume claims. + Capacity *resource.Quantity +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + Schedule *string +} diff --git a/controllers/provider-aws/pkg/apis/config/v1alpha1/types.go b/controllers/provider-aws/pkg/apis/config/v1alpha1/types.go index b98b236c8..14ad1a496 100644 --- a/controllers/provider-aws/pkg/apis/config/v1alpha1/types.go +++ b/controllers/provider-aws/pkg/apis/config/v1alpha1/types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to AWS-specific identifiers, i.e. AMIs. MachineImages []MachineImage `json:"machineImages,omitempty"` + // ETCD is the etcd configuration. + ETCD ETCD `json:"etcd"` } // MachineImage is a mapping from logical names and versions to AWS-specific identifiers, i.e. AMIs. @@ -46,3 +49,28 @@ type RegionAMIMapping struct { // AMI is the AMI for the machine image. AMI string `json:"ami"` } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage `json:"storage"` + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup `json:"backup"` +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + // +optional + ClassName *string `json:"className,omitempty"` + // Capacity is the storage capacity used in etcd-main volume claims. + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty"` +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + // +optional + Schedule *string `json:"schedule,omitempty"` +} diff --git a/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.conversion.go b/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.conversion.go index 73a6779f2..d45f8ad57 100644 --- a/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.conversion.go +++ b/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.conversion.go @@ -24,6 +24,7 @@ import ( unsafe "unsafe" config "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + resource "k8s.io/apimachinery/pkg/api/resource" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -45,6 +46,36 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ETCD)(nil), (*config.ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCD_To_config_ETCD(a.(*ETCD), b.(*config.ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCD)(nil), (*ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCD_To_v1alpha1_ETCD(a.(*config.ETCD), b.(*ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDBackup)(nil), (*config.ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(a.(*ETCDBackup), b.(*config.ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDBackup)(nil), (*ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(a.(*config.ETCDBackup), b.(*ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDStorage)(nil), (*config.ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(a.(*ETCDStorage), b.(*config.ETCDStorage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDStorage)(nil), (*ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(a.(*config.ETCDStorage), b.(*ETCDStorage), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MachineImage)(nil), (*config.MachineImage)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_MachineImage_To_config_MachineImage(a.(*MachineImage), b.(*config.MachineImage), scope) }); err != nil { @@ -70,6 +101,9 @@ func RegisterConversions(s *runtime.Scheme) error { func autoConvert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration(in *ControllerConfiguration, out *config.ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]config.MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_v1alpha1_ETCD_To_config_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -80,6 +114,9 @@ func Convert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration( func autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in *config.ControllerConfiguration, out *ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_config_ETCD_To_v1alpha1_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -88,6 +125,78 @@ func Convert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration( return autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in, out, s) } +func autoConvert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + if err := Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_ETCD_To_config_ETCD is an autogenerated conversion function. +func Convert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCD_To_config_ETCD(in, out, s) +} + +func autoConvert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + if err := Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_config_ETCD_To_v1alpha1_ETCD is an autogenerated conversion function. +func Convert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + return autoConvert_config_ETCD_To_v1alpha1_ETCD(in, out, s) +} + +func autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup is an autogenerated conversion function. +func Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in, out, s) +} + +func autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup is an autogenerated conversion function. +func Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + return autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in, out, s) +} + +func autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage is an autogenerated conversion function. +func Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in, out, s) +} + +func autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage is an autogenerated conversion function. +func Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + return autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in, out, s) +} + func autoConvert_v1alpha1_MachineImage_To_config_MachineImage(in *MachineImage, out *config.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 0697c44e3..c5d0dddfd 100644 --- a/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/controllers/provider-aws/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -35,6 +35,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -56,6 +57,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-aws/pkg/apis/config/zz_generated.deepcopy.go b/controllers/provider-aws/pkg/apis/config/zz_generated.deepcopy.go index 649488a3c..b94dc1cd3 100644 --- a/controllers/provider-aws/pkg/apis/config/zz_generated.deepcopy.go +++ b/controllers/provider-aws/pkg/apis/config/zz_generated.deepcopy.go @@ -35,6 +35,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -56,6 +57,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-aws/pkg/aws/types.go b/controllers/provider-aws/pkg/aws/types.go index 84ead4530..1ce91615b 100644 --- a/controllers/provider-aws/pkg/aws/types.go +++ b/controllers/provider-aws/pkg/aws/types.go @@ -19,6 +19,8 @@ import "path/filepath" const ( // Name is the name of the AWS provider. Name = "provider-aws" + // StorageProviderName is the name of the AWS storage provider. + StorageProviderName = "S3" // MachineControllerManagerImageName is the name of the MachineControllerManager image. MachineControllerManagerImageName = "machine-controller-manager" @@ -26,6 +28,8 @@ const ( TerraformerImageName = "terraformer" // HyperkubeImageName is the name of the hyperkube image. HyperkubeImageName = "hyperkube" + // ETCDBackupRestoreImageName is the name of the etcd backup and restore image. + ETCDBackupRestoreImageName = "etcd-backup-restore" // AccessKeyID is a constant for the key in a cloud provider secret and backup secret that holds the AWS access key id. AccessKeyID = "accessKeyID" @@ -33,6 +37,10 @@ const ( SecretAccessKey = "secretAccessKey" // Region is a constant for the key in a backup secret that holds the AWS region. Region = "region" + // BucketName is a constant for the key in a backup secret that holds the bucket name. + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + BucketName = "bucketName" // TerraformerPurposeInfra is a constant for the complete Terraform setup with purpose 'infrastructure'. TerraformerPurposeInfra = "infra" // VPCIDKey is the vpc_id tf state key @@ -58,6 +66,8 @@ const ( CloudProviderConfigName = "cloud-provider-config" // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" + // BackupSecretName is the name of the secret containing the credentials for storing the backups of Shoot clusters. + BackupSecretName = "etcd-backup" ) var ( diff --git a/controllers/provider-aws/pkg/cmd/config.go b/controllers/provider-aws/pkg/cmd/config.go index 0ab6cd6ab..40b1f47cf 100644 --- a/controllers/provider-aws/pkg/cmd/config.go +++ b/controllers/provider-aws/pkg/cmd/config.go @@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) { *machineImages = c.Config.MachineImages } +// ApplyETCDStorage sets the given etcd storage configuration to that of this Config. +func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { + *etcdStorage = c.Config.ETCD.Storage +} + +// ApplyETCDBackup sets the given etcd backup configuration to that of this Config. +func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) { + *etcdBackup = c.Config.ETCD.Backup +} + // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/controllers/provider-aws/pkg/cmd/options.go b/controllers/provider-aws/pkg/cmd/options.go index f2c5ba4b9..6d6960667 100644 --- a/controllers/provider-aws/pkg/cmd/options.go +++ b/controllers/provider-aws/pkg/cmd/options.go @@ -19,7 +19,8 @@ import ( infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/controller/worker" controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplane" - "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplaneexposure" + controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplanebackup" + controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/webhook/controlplaneexposure" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane" extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" @@ -37,11 +38,11 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { ) } -// WebhookAddToManagerOptions are the webhookcmd.AddToManagerOptions for the provider webhooks. -func WebhookAddToManagerOptions() *webhookcmd.AddToManagerOptions { - return webhookcmd.NewAddToManagerOptions( - "aws-webhooks", - webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.Factory), - webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposure.Factory), +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions( + webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager), ) } diff --git a/controllers/provider-aws/pkg/webhook/controlplane/add.go b/controllers/provider-aws/pkg/webhook/controlplane/add.go index 0fd03b827..fa32e58bb 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/add.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/add.go @@ -30,8 +30,9 @@ import ( var logger = log.Log.WithName("aws-controlplane-webhook") -// Factory creates a webhook. -func Factory(mgr manager.Manager) (webhook.Webhook, error) { +// AddToManager creates a webhook and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") return controlplane.Add(mgr, controlplane.AddArgs{ Kind: extensionswebhook.ShootKind, Provider: aws.Type, diff --git a/controllers/provider-aws/pkg/webhook/controlplane/ensurer.go b/controllers/provider-aws/pkg/webhook/controlplane/ensurer.go index de52c3786..b2cbae00b 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/ensurer.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/ensurer.go @@ -37,7 +37,7 @@ func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { } type ensurer struct { - controlplane.NoopEnsurer + genericmutator.NoopEnsurer logger logr.Logger } diff --git a/controllers/provider-aws/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-aws/pkg/webhook/controlplane/ensurer_test.go index 72e2a3290..ea16a8adc 100644 --- a/controllers/provider-aws/pkg/webhook/controlplane/ensurer_test.go +++ b/controllers/provider-aws/pkg/webhook/controlplane/ensurer_test.go @@ -38,7 +38,7 @@ func TestController(t *testing.T) { RunSpecs(t, "AWS Controlplane Webhook Suite") } -var _ = Describe("Mutator", func() { +var _ = Describe("Ensurer", func() { var ( ctrl *gomock.Controller ) diff --git a/controllers/provider-aws/pkg/webhook/controlplanebackup/add.go b/controllers/provider-aws/pkg/webhook/controlplanebackup/add.go new file mode 100644 index 000000000..d4d1da952 --- /dev/null +++ b/controllers/provider-aws/pkg/webhook/controlplanebackup/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/imagevector" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the AWS backup webhook to the manager. +type AddOptions struct { + // ETCDBackup is the etcd backup configuration. + ETCDBackup config.ETCDBackup +} + +var logger = log.Log.WithName("aws-controlplanebackup-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.BackupKind, + Provider: aws.Type, + Types: []runtime.Object{&appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDBackup, imagevector.ImageVector(), logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer.go b/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer.go new file mode 100644 index 000000000..30d438afd --- /dev/null +++ b/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer.go @@ -0,0 +1,136 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "github.com/gardener/gardener/pkg/operation/common" + + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdBackup *config.ETCDBackup, imageVector imagevector.ImageVector, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdBackup: etcdBackup, + imageVector: imageVector, + logger: logger.WithName("aws-controlplanebackup-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdBackup *config.ETCDBackup + imageVector imagevector.ImageVector + logger logr.Logger +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + if err := e.ensureContainers(&ss.Spec.Template.Spec, ss.Name, cluster); err != nil { + return err + } + return nil +} + +func (e *ensurer) ensureContainers(ps *corev1.PodSpec, name string, cluster *extensionscontroller.Cluster) error { + c, err := e.getBackupRestoreContainer(name, cluster) + if err != nil { + return err + } + ps.Containers = controlplane.EnsureContainerWithName(ps.Containers, *c) + return nil +} + +func (e *ensurer) getBackupRestoreContainer(name string, cluster *extensionscontroller.Cluster) (*corev1.Container, error) { + // Find etcd-backup-restore image + // TODO Get seed version from clientset when it's possible to inject it + image, err := e.imageVector.FindImage(aws.ETCDBackupRestoreImageName, "", cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, errors.Wrapf(err, "could not find image %s", aws.ETCDBackupRestoreImageName) + } + + const ( + defaultSchedule = "0 */24 * * *" + ) + + // Determine provider and container env variables + // They are only specified for the etcd-main stateful set (backup is enabled) + var ( + provider string + env []corev1.EnvVar + ) + if name == common.EtcdMainStatefulSetName { + provider = aws.StorageProviderName + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_REGION", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.Region, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.AccessKeyID, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.SecretAccessKey, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + } + } + + // Determine schedule + var schedule = defaultSchedule + if e.etcdBackup.Schedule != nil { + schedule = defaultSchedule + } + + return controlplane.GetBackupRestoreContainer(name, schedule, provider, image.String(), nil, env, nil), nil +} diff --git a/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer_test.go b/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer_test.go new file mode 100644 index 000000000..b54e95223 --- /dev/null +++ b/controllers/provider-aws/pkg/webhook/controlplanebackup/ensurer_test.go @@ -0,0 +1,207 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AWS Controlplane Backup Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + Describe("#EnsureETCDStatefulSet", func() { + var ( + etcdBackup = &config.ETCDBackup{ + Schedule: util.StringPtr("0 */24 * * *"), + } + + imageVector = imagevector.ImageVector{ + { + Name: aws.ETCDBackupRestoreImageName, + Repository: "test-repository", + Tag: "test-tag", + }, + } + + cluster = &extensionscontroller.Cluster{ + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + ) + + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + var ( + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_REGION", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.Region, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.AccessKeyID, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + { + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: aws.SecretAccessKey, + LocalObjectReference: corev1.LocalObjectReference{Name: aws.BackupSecretName}, + }, + }, + }, + } + ) + + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdMainStatefulSetName, "0 */24 * * *", aws.StorageProviderName, + "test-repository:test-tag", nil, env, nil))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdEventsStatefulSetName, "0 */24 * * *", "", + "test-repository:test-tag", nil, nil, nil))) +} diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go index 7c1c218ba..4bbacb39e 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/add.go @@ -15,6 +15,7 @@ package controlplaneexposure import ( + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/aws" extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" @@ -28,15 +29,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" ) +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the AWS exposure webhook to the manager. +type AddOptions struct { + // ETCDStorage is the etcd storage configuration. + ETCDStorage config.ETCDStorage +} + var logger = log.Log.WithName("aws-controlplaneexposure-webhook") -// Factory creates a webhook. -func Factory(mgr manager.Manager) (webhook.Webhook, error) { +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { logger.Info("Adding webhook to manager") return controlplane.Add(mgr, controlplane.AddArgs{ Kind: extensionswebhook.SeedKind, Provider: aws.Type, - Types: []runtime.Object{&corev1.Service{}, &appsv1.Deployment{}}, - Mutator: genericmutator.NewMutator(NewEnsurer(logger), nil, nil, logger), + Types: []runtime.Object{&corev1.Service{}, &appsv1.Deployment{}, &appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDStorage, logger), nil, nil, logger), }) } + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer.go index 446c77b26..3dad61a95 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer.go @@ -17,24 +17,29 @@ package controlplaneexposure import ( "context" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + "github.com/gardener/gardener/pkg/operation/common" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) // NewEnsurer creates a new controlplaneexposure ensurer. -func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { +func NewEnsurer(etcdStorage *config.ETCDStorage, logger logr.Logger) genericmutator.Ensurer { return &ensurer{ - logger: logger.WithName("aws-controlplaneexposure-ensurer"), + etcdStorage: etcdStorage, + logger: logger.WithName("aws-controlplaneexposure-ensurer"), } } type ensurer struct { - controlplane.NoopEnsurer - logger logr.Logger + genericmutator.NoopEnsurer + etcdStorage *config.ETCDStorage + logger logr.Logger } // EnsureKubeAPIServerService ensures that the kube-apiserver service conforms to the provider requirements. @@ -60,3 +65,22 @@ func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1 } return nil } + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + e.ensureVolumeClaimTemplates(&ss.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureVolumeClaimTemplates(spec *appsv1.StatefulSetSpec, name string) { + t := e.getVolumeClaimTemplate(name) + spec.VolumeClaimTemplates = controlplane.EnsurePVCWithName(spec.VolumeClaimTemplates, *t) +} + +func (e *ensurer) getVolumeClaimTemplate(name string) *corev1.PersistentVolumeClaim { + var etcdStorage config.ETCDStorage + if name == common.EtcdMainStatefulSetName { + etcdStorage = *e.etcdStorage + } + return controlplane.GetETCDVolumeClaimTemplate(name, etcdStorage.ClassName, etcdStorage.Capacity) +} diff --git a/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer_test.go b/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer_test.go index a5e4f5625..f1628798d 100644 --- a/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer_test.go +++ b/controllers/provider-aws/pkg/webhook/controlplaneexposure/ensurer_test.go @@ -18,6 +18,8 @@ import ( "context" "testing" + "github.com/gardener/gardener-extensions/controllers/provider-aws/pkg/apis/config" + "github.com/gardener/gardener-extensions/pkg/util" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener/pkg/operation/common" @@ -25,6 +27,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,7 +36,13 @@ func TestController(t *testing.T) { RunSpecs(t, "AWS Controlplane Exposure Webhook Suite") } -var _ = Describe("Mutator", func() { +var _ = Describe("Ensurer", func() { + var ( + etcdStorage = &config.ETCDStorage{ + ClassName: util.StringPtr("gardener.cloud-fast"), + Capacity: util.QuantityPtr(resource.MustParse("80Gi")), + } + ) Describe("#EnsureKubeAPIServerService", func() { It("should add annotations to kube-apiserver service", func() { @@ -44,7 +53,7 @@ var _ = Describe("Mutator", func() { ) // Create ensurer - ensurer := NewEnsurer(logger) + ensurer := NewEnsurer(etcdStorage, logger) // Call EnsureKubeAPIServerService method and check the result err := ensurer.EnsureKubeAPIServerService(context.TODO(), svc) @@ -80,7 +89,7 @@ var _ = Describe("Mutator", func() { ) // Create ensurer - ensurer := NewEnsurer(logger) + ensurer := NewEnsurer(etcdStorage, logger) // Call EnsureKubeAPIServerDeployment method and check the result err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) @@ -108,7 +117,7 @@ var _ = Describe("Mutator", func() { ) // Create ensurer - ensurer := NewEnsurer(logger) + ensurer := NewEnsurer(etcdStorage, logger) // Call EnsureKubeAPIServerDeployment method and check the result err := ensurer.EnsureKubeAPIServerDeployment(context.TODO(), dep) @@ -116,6 +125,102 @@ var _ = Describe("Mutator", func() { checkKubeAPIServerDeployment(dep) }) }) + + Describe("#EnsureETCDStatefulSet", func() { + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-main"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-events"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) }) func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { @@ -124,3 +229,14 @@ func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { Expect(c).To(Not(BeNil())) Expect(c.Command).To(ContainElement("--endpoint-reconciler-type=none")) } + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-main") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdMainStatefulSetName, util.StringPtr("gardener.cloud-fast"), + util.QuantityPtr(resource.MustParse("80Gi"))))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-events") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdEventsStatefulSetName, nil, nil))) +} diff --git a/controllers/provider-gcp/charts/images.yaml b/controllers/provider-gcp/charts/images.yaml index a53859762..987b12017 100644 --- a/controllers/provider-gcp/charts/images.yaml +++ b/controllers/provider-gcp/charts/images.yaml @@ -10,3 +10,7 @@ images: sourceRepository: github.com/gardener/machine-controller-manager repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager tag: "0.18.0" +- name: etcd-backup-restore + sourceRepository: github.com/gardener/etcd-backup-restore + repository: eu.gcr.io/gardener-project/gardener/etcdbrctl + tag: "0.6.3" diff --git a/controllers/provider-gcp/charts/provider-gcp/templates/configmap.yaml b/controllers/provider-gcp/charts/provider-gcp/templates/configmap.yaml index 2c4b971fd..34320948a 100644 --- a/controllers/provider-gcp/charts/provider-gcp/templates/configmap.yaml +++ b/controllers/provider-gcp/charts/provider-gcp/templates/configmap.yaml @@ -15,3 +15,9 @@ data: machineImages: {{ toYaml .Values.config.machineImages | indent 4 }} {{- end }} + etcd: + storage: + className: {{ .Values.config.etcd.storage.className }} + capacity: {{ .Values.config.etcd.storage.capacity }} + backup: + schedule: {{ .Values.config.etcd.backup.schedule }} diff --git a/controllers/provider-gcp/charts/provider-gcp/templates/storageclass.yaml b/controllers/provider-gcp/charts/provider-gcp/templates/storageclass.yaml new file mode 100644 index 000000000..7b6c043fb --- /dev/null +++ b/controllers/provider-gcp/charts/provider-gcp/templates/storageclass.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.config.etcd.storage.className }} +provisioner: kubernetes.io/gce-pd +allowVolumeExpansion: true +parameters: + type: pd-ssd \ No newline at end of file diff --git a/controllers/provider-gcp/charts/provider-gcp/values.yaml b/controllers/provider-gcp/charts/provider-gcp/values.yaml index 1a9b800af..8f05a5475 100644 --- a/controllers/provider-gcp/charts/provider-gcp/values.yaml +++ b/controllers/provider-gcp/charts/provider-gcp/values.yaml @@ -19,9 +19,16 @@ disableControllers: disableWebhooks: - controlplane - controlplaneexposure +- controlplanebackup config: machineImages: - name: coreos version: 2023.5.0 image: projects/coreos-cloud/global/images/coreos-stable-2023-5-0-v20190312 + etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go b/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go index 54e8bbe28..3994fdcdb 100644 --- a/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go +++ b/controllers/provider-gcp/cmd/gardener-extension-provider-gcp/app/app.go @@ -25,6 +25,8 @@ import ( gcpinfrastructure "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/infrastructure" gcpworker "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/worker" "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + gcpcontrolplanebackup "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplanebackup" + gcpcontrolplaneexposure "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplaneexposure" "github.com/gardener/gardener-extensions/pkg/controller" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" @@ -45,44 +47,49 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts = &gcpcmd.ConfigOptions{} + // options for the controlplane controller + controlPlaneCtrlOpts = &controllercmd.ControllerOptions{ + MaxConcurrentReconciles: 5, + } + + // options for the infrastructure controller infraCtrlOpts = &controllercmd.ControllerOptions{ MaxConcurrentReconciles: 5, } infraReconcileOpts = &infrastructure.ReconcilerOptions{ IgnoreOperationAnnotation: true, } - unprefixedInfraOpts = controllercmd.NewOptionAggregator(infraCtrlOpts, infraReconcileOpts) - - controlPlaneCtrlOpts = &controllercmd.ControllerOptions{ - MaxConcurrentReconciles: 5, - } + infraCtrlOptsUnprefixed = controllercmd.NewOptionAggregator(infraCtrlOpts, infraReconcileOpts) + // options for the worker controller workerCtrlOpts = &controllercmd.ControllerOptions{ MaxConcurrentReconciles: 5, } - controllerSwitches = gcpcmd.ControllerSwitchOptions() - webhookSwitches = gcpcmd.WebhookAddToManagerOptions() + controllerSwitches = gcpcmd.ControllerSwitchOptions() + webhookSwitches = gcpcmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("gcp-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, mgrOpts, - controllercmd.PrefixOption("infrastructure-", &unprefixedInfraOpts), controllercmd.PrefixOption("controlplane-", controlPlaneCtrlOpts), + controllercmd.PrefixOption("infrastructure-", &infraCtrlOptsUnprefixed), controllercmd.PrefixOption("worker-", workerCtrlOpts), + configFileOpts, controllerSwitches, - webhookSwitches, + webhookOptions, ) ) - webhookSwitches.Server = webhookcmd.ServerOptions{ - Port: 7890, - CertDir: "/tmp/cert", - Mode: webhookcmd.ServiceMode, - Name: "webhooks", - Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), - ServiceSelectors: "{}", - Host: "localhost", - } cmd := &cobra.Command{ Use: fmt.Sprintf("%s-controller-manager", gcp.Name), @@ -92,10 +99,6 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Error completing options") } - if err := configFileOpts.Complete(); err != nil { - controllercmd.LogErrAndExit(err, "Error completing config options") - } - mgr, err := manager.New(restOpts.Completed().Config, mgrOpts.Completed().Options()) if err != nil { controllercmd.LogErrAndExit(err, "Could not instantiate manager") @@ -110,6 +113,8 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } configFileOpts.Completed().ApplyMachineImages(&gcpworker.DefaultAddOptions.MachineImages) + configFileOpts.Completed().ApplyETCDStorage(&gcpcontrolplaneexposure.DefaultAddOptions.ETCDStorage) + configFileOpts.Completed().ApplyETCDBackup(&gcpcontrolplanebackup.DefaultAddOptions.ETCDBackup) controlPlaneCtrlOpts.Completed().Apply(&gcpcontrolplane.Options) infraCtrlOpts.Completed().Apply(&gcpinfrastructure.DefaultAddOptions.Controller) infraReconcileOpts.Completed().Apply(&gcpinfrastructure.DefaultAddOptions.IgnoreOperationAnnotation) @@ -119,7 +124,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } - if err := webhookSwitches.Completed().AddToManager(mgr); err != nil { + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") } @@ -130,7 +135,6 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { } aggOption.AddFlags(cmd.Flags()) - configFileOpts.AddFlags(cmd.Flags()) return cmd } diff --git a/controllers/provider-gcp/example/00-componentconfig.yaml b/controllers/provider-gcp/example/00-componentconfig.yaml index c39119eef..3d8105fef 100644 --- a/controllers/provider-gcp/example/00-componentconfig.yaml +++ b/controllers/provider-gcp/example/00-componentconfig.yaml @@ -5,3 +5,9 @@ machineImages: - name: coreos version: 2023.5.0 image: projects/coreos-cloud/global/images/coreos-stable-2023-5-0-v20190312 +etcd: + storage: + className: gardener.cloud-fast + capacity: 25Gi + backup: + schedule: "0 */24 * * *" diff --git a/controllers/provider-gcp/example/30-etcd-backup-secret.yaml b/controllers/provider-gcp/example/30-etcd-backup-secret.yaml new file mode 100644 index 000000000..7f85cc3ff --- /dev/null +++ b/controllers/provider-gcp/example/30-etcd-backup-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-backup + namespace: shoot--foo--bar +type: Opaque +data: + serviceaccount.json: eyJwcm9qZWN0X2lkIjoiYWJjIn0= + bucketName: YnVja2V0MTIz # bucket123 diff --git a/controllers/provider-gcp/example/controller-registration.yaml b/controllers/provider-gcp/example/controller-registration.yaml index 25745ab10..0d4d047fe 100644 --- a/controllers/provider-gcp/example/controller-registration.yaml +++ b/controllers/provider-gcp/example/controller-registration.yaml @@ -14,7 +14,7 @@ spec: deployment: type: helm providerConfig: - chart: H4sIAAAAAAAAA+0aXW/bOLLP/hUD4x66h0qynbjZ9aEPbprtGtemRpzdok8LWqJlNpKoJSm7vrb//YakJEuynY9tmmJbDQyYIjkfHHK+KKWCr1hAhRP6qffo60AP4WQ4NP8IzX/T7h8d9wfDwdOnur8/6A+PH8HwK8lTg0wqIgAeCc7VdfNuGv+HQlrd/9MlEcrdkDi6Vx437T/udmP/j/u9wSPo3asUB+AH33+Ssj+okIwnI1j1OyRNy8eee+L2nICuOgGVvmCpMt1j+I1GMfj6rMCCC1BLCi+JCGhCBbw8ncI0P1NAPyiaaGKdhMR0BNXD1lnt8vnWyvgBoWb/AffdkN87jxvsf4CjDfs/wmZr/w8BngenPN0IFi4VPPZ/gkGv/wvMxlOYnQEaN0nMA1ksWMSIouDzOCXJxoVxFIFBkyCopGJFAxcul0wCTqWA/xHz0fxpAFmivYH2E+OU+Pg34wu1JoLCKzvlCaxcGKC/8GmqgEhIuEI8jihizSRSSwz6q8np2TkKpjl0PA9/BYU9TErauUeDgduDx3pCNx/q/vQfTWLDM4jJRjOFDJmpchG5QMhdLxsVkPgU1kwtrTSWiqtpvMtp8LkiOJ0gQopPi+pEICoX2sBSqXTkeev12iVGYpeL0MuVJr18rQ5KnWP9nkRUam3/lTGBK55vAP01IpA5yhqRtdmwUFAcU1xLvRZMsSR8AjJXuCYTMKkEm2eqprRCRlx6dQKqDY9AdzyDyawLz8ezyeyJJvJ2cvnbm98v4e344mJ8fjk5m8GbCzh9c/5icjl5c45Pv8L4/B38d3L+4glQpncS1ZkKvQIUk2l14onRtGaU1kQogopMqc8WzMelJWFGQgohx6iR4IogpSJmUm+rRAEDTSZiMVNEma6ddbkdnBLyUaijlD7HruuVvyXxr7xixPF5ogSPInSKgoZaF4aoK5e1AAZuToN+ILgY6h3C0/kUTJKFINiV+SoTdKTxT+38Ka7Odrzl4ooK3dTCwhSl0ou2kZYmepclVNcgszTleRTOO7Vu9LJ9LgT1FWyFgppQnbRKvY28PyrU4r+ieJDxYMn7rQTvXv8d93X8b+u/rw8H9h/dxoKFMUnvoxq8af9PTpr130kPS8I2/3sAaNR/VywJRjoy4e6/JmknpooERJFRB8DWcGFe6TllcedUz5BTnpwcQ2KgQbSPH8G9oBElGIrPi274/BlnRWROI6k5gM5n3KtsjkGe4jF0GfduxfUALktwc5N97DXnYl1WYnPQR/DJkHIcJye51Q6ycQuebimGdHP0QkLXj3gWeKs+idIl6RsypVbzUGz1m9lQ3Pn40QG2APcPEmW0pBdjRsgSOokxSEurKIBa5wgxMc97p9OLa5E/YSqIsik41nQ0O5oEuvnooP0HNI34Jkake3AAN9j/8Hhw1LT/waC1/weB6glHA5Je6QRelEfg7l7gwW1/SaMYE3TPZLNfx13oakQLKqgpuaSdldtd3nnKMzQzszKJ6L7iYpQbrvKXryqL/bLl3l16gMK4c4EqW2roJVj92uKp6ELPuKT+lczibTrg3M79GxlY4kdZQOFxKhhq5V/uZS6B+xwlmxKsoruNRKP7E1ZVn0AuyWD4FBkXfq9+UL5ce39HfwDFCTBtKlZYWY59X2/5+R1464qMoHMW5WqcWxuVBabd+gi6lfNnuvQp5JLhodugyKOdYUVC7O/W6UyzKJpyPLyb2oG2GGk5uN0JvYI4xpp7uxkOeHtkX26wSK/MaZ6TolhGWsirOtPJj5GjL2GeeVT5++jXQpdF8CqxvEaP1cpvZPlBs/AzLJITheW6fkBe8llFBVsRUR01fHeLO9skvqwqZw83FiYc/3hKbcR3tsZ2S36WwpuCwLjEb3JemyuEu6/P4t20roBJfQlR2bwawXx4m+fo1OM9Zwl0n3QP0VrT+ZLzq32E3uZD11DJsYvzEvOAPstN87p52t6e6WNYcL9psglZz64JZAexc2mcIh7gQrv7nVd31L3hlKMCDvstRN/juLpb4Wiyqpqs9TmvzsYvzi7+PHt1dqrv7f48H78+m03Hp2flTICV3pVfBY9HlU6ABaNRcEEX9d68X7v3URll3FKH5Vx9cSV35Sm0Zy6URYVy6TanXEf4k59/6VVGUUuK+xwT+MvTadkvqOSZ8Kmsiqjz30bWXM7Dk5bkuXJ/UN3YFY+ymL7Wvn6P0HarKzxiPdGq4A7ea5cuFeoAVRUj1nbYyrcTURp0/aKuq+rj7mVdAQFdkCxSr9HoRnA86DWZV6WX1BdUVfnanjxy7jPFPQx0Xba3Yp0ZYvsS1Trl22Sl3zolf1A4UP+JOfHv7UOAG+q/o8HwuFH/DU/09x9t/ff1oWlRZuNJppZcsP/ZVwVXP5s4s70dilBnVFzwiP6tyvAfVPOJLNJO1dEXQS8Fz1IjtgOV65/6vU+nEXQc8K22pHmop3V7+zyUR2V2KM+zUv2GaE9PdapN4Grt7TBG0nkuTkiV+Y+YtI21rkxNKy1bWYp7SneX3e3uWV8RGGRlzPhTO16LIqhd3TSpkL762ivauinHVrj9Ejnm5sI05iXuwYNsEYL87WHtpVh1Qsoqm1wONBZfhhPLna4wc7BNG93y7iRIMX9VcldhDmxv2gpEky/WHoitMmX1QOEpoDsdczRQloS2fztjZ+g9n9tGyoNtw4t4aB7iTJk3iXng9KvXlTlPZMnjQhsYp1nCitHKlnb/3d3ds/xy0pUkNTazV7Mac5fUFzmr51YD373PwqXmWXmh+Gs01Slvpyte/bZ6kdn8PVqy8ZBFHla9FrmDgtvE7NvAgfyv7ni+MBO86f3f8bBfz/8Gvd7waZv/PQQcqKZqVtxe/3+/zqdm/yt7HXLfH4DfYP99dADN7z+OjtvvPx8E7LsEk38V7w5GQDM39IW2idKS8JzoUF92XHfhr0g4AhNHdAaQVl4wTBbnXE3156LoVjrblA8+fu50KrfK+Wv5stSxHqJxOz2CIXbXy6drJuLUQ7foI1iQSOq8x5ZOB6l0dq+4de5TFbVyOd5p3GM3pzYe6QfcAVyD0YR+idbZeeFfvdkTlNu7svJD+kFvcOQOXXsPl78kyvdNf9GjERz7dUIY8TmJPDOnHEJD0BfymoozdHrOSn8K3DvqD75X39dCCy200EILLbTQQgsttNBCCy200EILLbTQQgsttNBCC98b/B8A+Y5MAFAAAA== + chart: H4sIAAAAAAAAA+0ba3PTuJbP+RVnMvcDMNh5l93c4UMoXbZzoWSaLgyfdhRbSUQdy1eSE3qB/36PHnZsJ2napZQBfGAmtqTz0NHReUhuIviKhVR48yBpPfg20EZ4OhiYX4Tqr3nu9Pqd7qB7dKTbO93OoP8ABt9InhKkUhEB8EBwrq4bd6j/B4WkuP7HCyKUf0WW0Z3yOLT+uNqV9e932t0H0L5TKfbAL77+JGFvqZCMx0NYdRokSfLXtv/Ub3shXTVCKgPBEmWaR/AnjZYQaFuBGRegFhReEhHSmAp4eTyGsbMpoB8VjTWxRkyWdAhFY2ustvl8b2X8glDa/yEP/Dm/cx4H9n8Xeyv7v4eP9f6/D2i14JgnV4LNFwoeBo+g2+78DpPRGCYngJubxOaFzGYsYkRRCPgyIfGVD6MoAoMmQVBJxYqGPlwsmAQcSgF/Ixbg9qchpLH2BtpPjBIS4M+Ez9SaCAqv7JAnsPKhi/4ioIkCIiHmCvE4oog1k0gtNuivTo9PzlAwzaHRauH/jMIOJjlt59Gg67fhoR7QdF3NR//WJK54CktypZlCisxUPgknEHLX00YFxAGFNVMLK42l4msa7x0NPlUEhxNESPBtVhwIRDmhDSyUSoat1nq99omR2Odi3nJKky03Vw+ldlh/xRGVWtv/TZnAGU+vAP01IpApyhqRtVmwuaDYp7iWei2YYvH8CUincE0mZFIJNk1VSWmZjDj14gBUG5pAczSB00kTno8mp5Mnmsi704s/3/x1Ae9G5+ejs4vTkwm8OYfjN2cvTi9O35zh2x8wOnsP/zk9e/EEKNMriepMhJ4Bism0OtFiNK0JpSURsqAiExqwGQtwavE8JXMKc45RI8YZQULFkkm9rBIFDDWZiC2ZIso0bc3Lb+CQOR/OdZTSduz7rfz/ggSXrazHC3isBI8idIqCzrUuDFFfLkoBDHxHg34kOBna2oen8yk4jWeCYFMaqFTQocY/tuPHODvb8I6LSyr0oxYWxiiVnrSNtDTWqyyhOAeZJgl3Udg1at3oaQdcCBoo2AgFJaEaSZF6HXl/VSjFf0XRkNGw5N1Wgrev//odHf/r+u/bw571R7cxY/MlSe6iGjyw/p1er19Z/6ft9qDO/+4DKvXfJYvDoY5MuPqvSdJYUkVCosiwAWBruLmr9Ly8uPOKNuTlluMwJAYaRPv0CfxzGlGCofgsa4YvX3BURKY0kpoD6HzGv0ynGOQpmqHPeOtGXPfgshgXN97FXnPO5mUlNoY+hM+GlOd5juRGO8jGz3j6uRjSd+iZhH4Q8TRsrTokShakY8jkWnWh2Oo3taG48emTB2wG/lsSpTSnt8SMkMX0dIlBWlpFAZQah4iJed57nV5ci/wZU0GUTUFf09HsaBxmJKkKQqt7AKm4QIzsFVUTESnPzApoDZaZaEzfofj5yIyuQdd5LVNXh7HdwA3yFNOTNNlIIjE/DtNovyAWwc/GaUrfe2/9CLDH/4c0ifjVEo3mDgLAAf8/6Hd7Vf/f7db+/16g6OHQgcpWHgRe5CZw+yhw775/QaMlFmgtU818m3Chq1EtqKCm5JYlT+Qaj3mKbtbMTCJ6gO5t6By3ChavCpP9uuneXnqAbHM7gQpLaujFMXfFc8H9L2hwKdPlJh30bhb+jQwsDqI0pPAwEQy18i//wkngP0fJxkQtoFlJNJuPsKr+DHJBuoMjZLyJB1FJd1+rvX+iP4DMAswzFSsW0FEQ6CU/uwVvXZETDM4in413401lgemwPoRmwf5Mk7ZCLhkanQ6kw61uRebY3izTGadRNOZovOUobTGSvLMU1vlySeJwsxgetHbIvrhKqCiMqdpJdliCtJBXcaTnzMjTh3DPWhjfd9EvhS6L0CrkciV6rHT8giw/ahZBKgT6N09Q/YK85LNyduFERHWU8P0N7uQqDmRROTu4sXnM8Ycn1GZ83maz3ZCfpfAmIzDK8auc1+YI6fbzs3iH5hUyqQ+hCotXIui6N3muTj0/cBZD80lzH601nS44v9xF6J3ruoaKw87sZclD+sxtzevG6f32TJthxv3QYBOynl0TyPZiO2m8LB7gRJu7nVdz2Dxg5aiA/X4L0Xc4ruZGOBqvilvW+pxXJ6MXJ+d/n7w6Odbntn+fjV6fTMaj45N8JMBKr8ofgi+HhUaAGaNReE5n5VbXrt37MI8yfq7DfKw+uJTb8mTaMxcKokA5d5tjriP8099+bxd6UUuKBxwLuIvjcd4uqOSpCKgsiqjrn0rVlI9DS4tdrdTpFhd2xaN0SV9rX79DaLvUBR5LPdCq4Bbea5suFWoPVbVErE23lW8rolToBlldX9TH7cv6DEI6I2mkXuOmG0K/264yL0ovaSCoKvK1LS5y7tqKOxjounznicXEENuVqJYp3yQr/d4p+b3CnvpPYDl9Zx+CHKj/et1B9fxv8FR//1PXf98eqjvKLDxJ1YIL9j97VXT5m4kzm9PBCHVGxTmP6D+qDH+gmk+kkXaqnj4IfCl4mhixPSgc/5XP/RqVoONBYLUlzUs5rdvZ1kJ5VGq7XJ6V6BvCHS3FoTaBKz1vujGSTp04c6rMb8SkfVjrytQ8JflTmuCa0u1pN5s75pcFBlnoM/7U9peiCGpXP5pUSB997hRtXZVjI9xuiTxzcmEepjnuXkO2CKG7PS5dihYHJKywyHlHZfJ5OLHc6QozB/too5trjsME81cltxXmweakLUM0+WLphdgqUxYNCq2AbjVMcYOyeG7bNyO2uj7wqX1IeLh5aEV8bl6WqTI3yS5wBsXjascTWfJlpg2M0yxmWW9hSZuPm9tr5g6nfUkSs2d2alZjbpP6Kmf13Grgp/dZOFWXlWeKv0ZTjfx2ouDVb6oXmU4/4E42HjLLw4rHIrdQcJ2YfR/Yk/+VHc9XZoKH7v/7g045/+u224OjOv+7D9hTTZV2cX38//M6n337317Mmlvdr64DD+3/Xvuoev836NXf/94LFPd/dhtfzaEmtv1YG8MOZ3CbLwOMuWluVAyhvP3mAfWSsEGiiK/fmuOsk48Jia1oWB7RRkIEklHu5kRdJfpPCkJPyvB7a/HHhdL+X9lVvOs/ADr0/Vf3qHr+0+/16/v/ewF7l2jqr+zucAg09eeBMJsyi6RoJzrVzxuuu/BTZD4EE0d0BZAULhhPZ2dcjfWfC2Ba0diUfPDpS6NRuFVyn2XlRx02Q6jcTg1hgM3l45NrBuLQfbdoQ5iRSOq6xx6d7KXS2L7i0rVPUdTC5Vijco9VHVp5pR9xBXAOlWb7YZNRj75Zb2x9BVY87heU2wP0/K+ruu1uzx/49nDe3Ry7xdSfeWoEz36yNo/4lEQtMybvwt2hb+k0FW/gtb2V/vuQdq/TbRQ/Hat8OFb4bKx8PObNiMzuBDYfh3UHL5lpLH/0tfnkq9mGx61uHx7rf82fNRWroYYaaqihhhpqqKGGGmqooYYaaqihhhpquFP4PxDzcusAUAAA values: image: tag: 0.7.0-dev diff --git a/controllers/provider-gcp/pkg/apis/config/types.go b/controllers/provider-gcp/pkg/apis/config/types.go index f94512d2b..c291a94e4 100644 --- a/controllers/provider-gcp/pkg/apis/config/types.go +++ b/controllers/provider-gcp/pkg/apis/config/types.go @@ -15,6 +15,7 @@ package config import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to GCP-specific identifiers. MachineImages []MachineImage + // ETCD is the etcd configuration. + ETCD ETCD } // MachineImage is a mapping from logical names and versions to GCP-specific identifiers. @@ -38,3 +41,25 @@ type MachineImage struct { // Image is the path to the image. Image string } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + ClassName *string + // Capacity is the storage capacity used in etcd-main volume claims. + Capacity *resource.Quantity +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + Schedule *string +} diff --git a/controllers/provider-gcp/pkg/apis/config/v1alpha1/types.go b/controllers/provider-gcp/pkg/apis/config/v1alpha1/types.go index 3f4b5c1a8..ff4f4f2c4 100644 --- a/controllers/provider-gcp/pkg/apis/config/v1alpha1/types.go +++ b/controllers/provider-gcp/pkg/apis/config/v1alpha1/types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,6 +28,8 @@ type ControllerConfiguration struct { // MachineImages is the list of machine images that are understood by the controller. It maps // logical names and versions to GCP-specific identifiers. MachineImages []MachineImage `json:"machineImages,omitempty"` + // ETCD is the etcd configuration. + ETCD ETCD `json:"etcd"` } // MachineImage is a mapping from logical names and versions to GCP-specific identifiers. @@ -38,3 +41,28 @@ type MachineImage struct { // Image is the path to the image. Image string `json:"image"` } + +// ETCD is an etcd configuration. +type ETCD struct { + // ETCDStorage is the etcd storage configuration. + Storage ETCDStorage `json:"storage"` + // ETCDBackup is the etcd backup configuration. + Backup ETCDBackup `json:"backup"` +} + +// ETCDStorage is an etcd storage configuration. +type ETCDStorage struct { + // ClassName is the name of the storage class used in etcd-main volume claims. + // +optional + ClassName *string `json:"className,omitempty"` + // Capacity is the storage capacity used in etcd-main volume claims. + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty"` +} + +// ETCDBackup is an etcd backup configuration. +type ETCDBackup struct { + // Schedule is the etcd backup schedule. + // +optional + Schedule *string `json:"schedule,omitempty"` +} diff --git a/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.conversion.go b/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.conversion.go index ed8a8d812..a7dbc0998 100644 --- a/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.conversion.go +++ b/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.conversion.go @@ -24,6 +24,7 @@ import ( unsafe "unsafe" config "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" + resource "k8s.io/apimachinery/pkg/api/resource" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -45,6 +46,36 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ETCD)(nil), (*config.ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCD_To_config_ETCD(a.(*ETCD), b.(*config.ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCD)(nil), (*ETCD)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCD_To_v1alpha1_ETCD(a.(*config.ETCD), b.(*ETCD), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDBackup)(nil), (*config.ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(a.(*ETCDBackup), b.(*config.ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDBackup)(nil), (*ETCDBackup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(a.(*config.ETCDBackup), b.(*ETCDBackup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ETCDStorage)(nil), (*config.ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(a.(*ETCDStorage), b.(*config.ETCDStorage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ETCDStorage)(nil), (*ETCDStorage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(a.(*config.ETCDStorage), b.(*ETCDStorage), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MachineImage)(nil), (*config.MachineImage)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_MachineImage_To_config_MachineImage(a.(*MachineImage), b.(*config.MachineImage), scope) }); err != nil { @@ -60,6 +91,9 @@ func RegisterConversions(s *runtime.Scheme) error { func autoConvert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration(in *ControllerConfiguration, out *config.ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]config.MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_v1alpha1_ETCD_To_config_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -70,6 +104,9 @@ func Convert_v1alpha1_ControllerConfiguration_To_config_ControllerConfiguration( func autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in *config.ControllerConfiguration, out *ControllerConfiguration, s conversion.Scope) error { out.MachineImages = *(*[]MachineImage)(unsafe.Pointer(&in.MachineImages)) + if err := Convert_config_ETCD_To_v1alpha1_ETCD(&in.ETCD, &out.ETCD, s); err != nil { + return err + } return nil } @@ -78,6 +115,78 @@ func Convert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration( return autoConvert_config_ControllerConfiguration_To_v1alpha1_ControllerConfiguration(in, out, s) } +func autoConvert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + if err := Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_ETCD_To_config_ETCD is an autogenerated conversion function. +func Convert_v1alpha1_ETCD_To_config_ETCD(in *ETCD, out *config.ETCD, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCD_To_config_ETCD(in, out, s) +} + +func autoConvert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + if err := Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(&in.Storage, &out.Storage, s); err != nil { + return err + } + if err := Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(&in.Backup, &out.Backup, s); err != nil { + return err + } + return nil +} + +// Convert_config_ETCD_To_v1alpha1_ETCD is an autogenerated conversion function. +func Convert_config_ETCD_To_v1alpha1_ETCD(in *config.ETCD, out *ETCD, s conversion.Scope) error { + return autoConvert_config_ETCD_To_v1alpha1_ETCD(in, out, s) +} + +func autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup is an autogenerated conversion function. +func Convert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in *ETCDBackup, out *config.ETCDBackup, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDBackup_To_config_ETCDBackup(in, out, s) +} + +func autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + out.Schedule = (*string)(unsafe.Pointer(in.Schedule)) + return nil +} + +// Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup is an autogenerated conversion function. +func Convert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in *config.ETCDBackup, out *ETCDBackup, s conversion.Scope) error { + return autoConvert_config_ETCDBackup_To_v1alpha1_ETCDBackup(in, out, s) +} + +func autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage is an autogenerated conversion function. +func Convert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in *ETCDStorage, out *config.ETCDStorage, s conversion.Scope) error { + return autoConvert_v1alpha1_ETCDStorage_To_config_ETCDStorage(in, out, s) +} + +func autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + out.ClassName = (*string)(unsafe.Pointer(in.ClassName)) + out.Capacity = (*resource.Quantity)(unsafe.Pointer(in.Capacity)) + return nil +} + +// Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage is an autogenerated conversion function. +func Convert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in *config.ETCDStorage, out *ETCDStorage, s conversion.Scope) error { + return autoConvert_config_ETCDStorage_To_v1alpha1_ETCDStorage(in, out, s) +} + func autoConvert_v1alpha1_MachineImage_To_config_MachineImage(in *MachineImage, out *config.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index e8748f09d..3b508b3d5 100644 --- a/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/controllers/provider-gcp/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -33,6 +33,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { *out = make([]MachineImage, len(*in)) copy(*out, *in) } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -54,6 +55,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-gcp/pkg/apis/config/zz_generated.deepcopy.go b/controllers/provider-gcp/pkg/apis/config/zz_generated.deepcopy.go index d96f9c710..a0d0b5fca 100644 --- a/controllers/provider-gcp/pkg/apis/config/zz_generated.deepcopy.go +++ b/controllers/provider-gcp/pkg/apis/config/zz_generated.deepcopy.go @@ -33,6 +33,7 @@ func (in *ControllerConfiguration) DeepCopyInto(out *ControllerConfiguration) { *out = make([]MachineImage, len(*in)) copy(*out, *in) } + in.ETCD.DeepCopyInto(&out.ETCD) return } @@ -54,6 +55,71 @@ func (in *ControllerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCD) DeepCopyInto(out *ETCD) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Backup.DeepCopyInto(&out.Backup) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCD. +func (in *ETCD) DeepCopy() *ETCD { + if in == nil { + return nil + } + out := new(ETCD) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDBackup) DeepCopyInto(out *ETCDBackup) { + *out = *in + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDBackup. +func (in *ETCDBackup) DeepCopy() *ETCDBackup { + if in == nil { + return nil + } + out := new(ETCDBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStorage) DeepCopyInto(out *ETCDStorage) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStorage. +func (in *ETCDStorage) DeepCopy() *ETCDStorage { + if in == nil { + return nil + } + out := new(ETCDStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = *in diff --git a/controllers/provider-gcp/pkg/cmd/config.go b/controllers/provider-gcp/pkg/cmd/config.go index f758ce031..6df29dd2b 100644 --- a/controllers/provider-gcp/pkg/cmd/config.go +++ b/controllers/provider-gcp/pkg/cmd/config.go @@ -75,6 +75,16 @@ func (c *Config) ApplyMachineImages(machineImages *[]config.MachineImage) { *machineImages = c.Config.MachineImages } +// ApplyETCDStorage sets the given etcd storage configuration to that of this Config. +func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { + *etcdStorage = c.Config.ETCD.Storage +} + +// ApplyETCDBackup sets the given etcd backup configuration to that of this Config. +func (c *Config) ApplyETCDBackup(etcdBackup *config.ETCDBackup) { + *etcdBackup = c.Config.ETCD.Backup +} + // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/controllers/provider-gcp/pkg/cmd/options.go b/controllers/provider-gcp/pkg/cmd/options.go index 02fb0d613..e70fc37a8 100644 --- a/controllers/provider-gcp/pkg/cmd/options.go +++ b/controllers/provider-gcp/pkg/cmd/options.go @@ -19,7 +19,8 @@ import ( infrastructurecontroller "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/controller/worker" controlplanewebhook "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplane" - "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplaneexposure" + controlplanebackupwebhook "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplanebackup" + controlplaneexposurewebhook "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/webhook/controlplaneexposure" controllercmd "github.com/gardener/gardener-extensions/pkg/controller/cmd" extensionscontrolplanecontroller "github.com/gardener/gardener-extensions/pkg/controller/controlplane" extensionsinfrastructurecontroller "github.com/gardener/gardener-extensions/pkg/controller/infrastructure" @@ -37,11 +38,11 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { ) } -// WebhookAddToManagerOptions are the webhookcmd.AddToManagerOptions for the provider webhooks. -func WebhookAddToManagerOptions() *webhookcmd.AddToManagerOptions { - return webhookcmd.NewAddToManagerOptions( - "gcp-webhooks", +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions( webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager), - webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposure.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager), + webhookcmd.Switch(extensioncontrolplanewebhook.BackupWebhookName, controlplanebackupwebhook.AddToManager), ) } diff --git a/controllers/provider-gcp/pkg/gcp/types.go b/controllers/provider-gcp/pkg/gcp/types.go index 3ed0a3197..6fe91298a 100644 --- a/controllers/provider-gcp/pkg/gcp/types.go +++ b/controllers/provider-gcp/pkg/gcp/types.go @@ -19,11 +19,15 @@ import "path/filepath" const ( // Name is the name of the GCP provider. Name = "provider-gcp" + // StorageProviderName is the name of the GCP storage provider. + StorageProviderName = "GCS" // HyperkubeImageName is the name of the hyperkube image. HyperkubeImageName = "hyperkube" // MachineControllerManagerImageName is the name of the MachineControllerManager image. MachineControllerManagerImageName = "machine-controller-manager" + // ETCDBackupRestoreImageName is the name of the etcd backup and restore image. + ETCDBackupRestoreImageName = "etcd-backup-restore" // ServiceAccountJSONField is the field in a secret where the service account JSON is stored at. ServiceAccountJSONField = "serviceaccount.json" @@ -31,8 +35,15 @@ const ( // ServiceAccountJSONMCM is the field in a machine class secret where the service account JSON is stored at. ServiceAccountJSONMCM = "serviceAccountJSON" + // BucketName is a constant for the key in a backup secret that holds the bucket name. + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + BucketName = "bucketName" + // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" + // BackupSecretName is the name of the secret containing the credentials for storing the backups of Shoot clusters. + BackupSecretName = "etcd-backup" ) var ( diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/add.go b/controllers/provider-gcp/pkg/webhook/controlplane/add.go index e11ec819a..01e38d243 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplane/add.go +++ b/controllers/provider-gcp/pkg/webhook/controlplane/add.go @@ -30,7 +30,7 @@ import ( var logger = log.Log.WithName("gcp-controlplane-webhook") -// AddToManager adds a webhook to the given manager. +// AddToManager creates a webhook and adds it to the manager. func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { logger.Info("Adding webhook to manager") return controlplane.Add(mgr, controlplane.AddArgs{ diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/ensurer.go b/controllers/provider-gcp/pkg/webhook/controlplane/ensurer.go index 24237f835..9e2b2e85b 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplane/ensurer.go +++ b/controllers/provider-gcp/pkg/webhook/controlplane/ensurer.go @@ -39,7 +39,7 @@ func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { } type ensurer struct { - controlplane.NoopEnsurer + genericmutator.NoopEnsurer logger logr.Logger } diff --git a/controllers/provider-gcp/pkg/webhook/controlplane/ensurer_test.go b/controllers/provider-gcp/pkg/webhook/controlplane/ensurer_test.go index 40aa5e4be..8ad7846f3 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplane/ensurer_test.go +++ b/controllers/provider-gcp/pkg/webhook/controlplane/ensurer_test.go @@ -38,7 +38,7 @@ func TestController(t *testing.T) { RunSpecs(t, "GCP Controlplane Webhook Suite") } -var _ = Describe("Mutator", func() { +var _ = Describe("Ensurer", func() { var ( ctrl *gomock.Controller ) @@ -46,6 +46,7 @@ var _ = Describe("Mutator", func() { BeforeEach(func() { ctrl = gomock.NewController(GinkgoT()) }) + AfterEach(func() { ctrl.Finish() }) diff --git a/controllers/provider-gcp/pkg/webhook/controlplanebackup/add.go b/controllers/provider-gcp/pkg/webhook/controlplanebackup/add.go new file mode 100644 index 000000000..87b87dd74 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplanebackup/add.go @@ -0,0 +1,59 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/imagevector" + extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) + +// AddOptions are options to apply when adding the AWS backup webhook to the manager. +type AddOptions struct { + // ETCDBackup is the etcd backup configuration. + ETCDBackup config.ETCDBackup +} + +var logger = log.Log.WithName("gcp-controlplanebackup-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { + logger.Info("Adding webhook to manager") + return controlplane.Add(mgr, controlplane.AddArgs{ + Kind: extensionswebhook.BackupKind, + Provider: gcp.Type, + Types: []runtime.Object{&appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDBackup, imagevector.ImageVector(), logger), nil, nil, logger), + }) +} + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer.go b/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer.go new file mode 100644 index 000000000..f027bc1b7 --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer.go @@ -0,0 +1,137 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "path" + + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" + + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// NewEnsurer creates a new controlplaneexposure ensurer. +func NewEnsurer(etcdBackup *config.ETCDBackup, imageVector imagevector.ImageVector, logger logr.Logger) genericmutator.Ensurer { + return &ensurer{ + etcdBackup: etcdBackup, + imageVector: imageVector, + logger: logger.WithName("gcp-controlplanebackup-ensurer"), + } +} + +type ensurer struct { + genericmutator.NoopEnsurer + etcdBackup *config.ETCDBackup + imageVector imagevector.ImageVector + logger logr.Logger +} + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + if err := e.ensureContainers(&ss.Spec.Template.Spec, ss.Name, cluster); err != nil { + return err + } + e.ensureVolumes(&ss.Spec.Template.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureContainers(ps *corev1.PodSpec, name string, cluster *extensionscontroller.Cluster) error { + c, err := e.getBackupRestoreContainer(name, cluster) + if err != nil { + return err + } + ps.Containers = controlplane.EnsureContainerWithName(ps.Containers, *c) + return nil +} + +func (e *ensurer) getBackupRestoreContainer(name string, cluster *extensionscontroller.Cluster) (*corev1.Container, error) { + // Find etcd-backup-restore image + // TODO Get seed version from clientset when it's possible to inject it + image, err := e.imageVector.FindImage(gcp.ETCDBackupRestoreImageName, "", cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, errors.Wrapf(err, "could not find image %s", gcp.ETCDBackupRestoreImageName) + } + + const ( + mountPath = "/root/.gcp/" + defaultSchedule = "0 */24 * * *" + ) + + // Determine provider, container env variables, and volume mounts + // They are only specified for the etcd-main stateful set (backup is enabled) + var ( + provider string + env []corev1.EnvVar + volumeMounts []corev1.VolumeMount + ) + if name == common.EtcdMainStatefulSetName { + provider = gcp.StorageProviderName + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + // The bucket name is written to the backup secret by Gardener as a temporary solution. + // TODO In the future, the bucket name should come from a BackupBucket resource (see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md) + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: gcp.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: gcp.BackupSecretName}, + }, + }, + }, + { + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: path.Join(mountPath, gcp.ServiceAccountJSONField), + }, + } + volumeMounts = []corev1.VolumeMount{ + { + Name: gcp.BackupSecretName, + MountPath: mountPath, + }, + } + } + + // Determine schedule + var schedule = defaultSchedule + if e.etcdBackup.Schedule != nil { + schedule = defaultSchedule + } + + return controlplane.GetBackupRestoreContainer(name, schedule, provider, image.String(), nil, env, volumeMounts), nil +} + +func (e *ensurer) ensureVolumes(ps *corev1.PodSpec, name string) { + if name == common.EtcdMainStatefulSetName { + etcdBackupSecretVolume := corev1.Volume{ + Name: gcp.BackupSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: gcp.BackupSecretName, + }, + }, + } + ps.Volumes = controlplane.EnsureVolumeWithName(ps.Volumes, etcdBackupSecretVolume) + } +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer_test.go b/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer_test.go new file mode 100644 index 000000000..59d7a6eaa --- /dev/null +++ b/controllers/provider-gcp/pkg/webhook/controlplanebackup/ensurer_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplanebackup + +import ( + "context" + "path" + "testing" + + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/gardener/gardener-extensions/pkg/util" + "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" + + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" + "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "GCP Controlplane Backup Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + Describe("#EnsureETCDStatefulSet", func() { + var ( + etcdBackup = &config.ETCDBackup{ + Schedule: util.StringPtr("0 */24 * * *"), + } + + imageVector = imagevector.ImageVector{ + { + Name: gcp.ETCDBackupRestoreImageName, + Repository: "test-repository", + Tag: "test-tag", + }, + } + + cluster = &extensionscontroller.Cluster{ + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + ) + + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "backup-restore", + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdBackup, imageVector, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, cluster) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) +}) + +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + var ( + env = []corev1.EnvVar{ + { + Name: "STORAGE_CONTAINER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: gcp.BucketName, + LocalObjectReference: corev1.LocalObjectReference{Name: gcp.BackupSecretName}, + }, + }, + }, + { + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: path.Join("/root/.gcp/", gcp.ServiceAccountJSONField), + }, + } + volumeMounts = []corev1.VolumeMount{ + { + Name: gcp.BackupSecretName, + MountPath: "/root/.gcp/", + }, + } + etcdBackupSecretVolume = corev1.Volume{ + Name: gcp.BackupSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: gcp.BackupSecretName, + }, + }, + } + ) + + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdMainStatefulSetName, "0 */24 * * *", gcp.StorageProviderName, + "test-repository:test-tag", nil, env, volumeMounts))) + Expect(ss.Spec.Template.Spec.Volumes).To(ContainElement(etcdBackupSecretVolume)) + +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + c := controlplane.ContainerWithName(ss.Spec.Template.Spec.Containers, "backup-restore") + Expect(c).To(Equal(controlplane.GetBackupRestoreContainer(common.EtcdEventsStatefulSetName, "0 */24 * * *", "", + "test-repository:test-tag", nil, nil, nil))) + Expect(ss.Spec.Template.Spec.Volumes).To(BeEmpty()) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go index d025728c5..06ebe997f 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/add.go @@ -15,6 +15,7 @@ package controlplaneexposure import ( + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/gcp" extensionswebhook "github.com/gardener/gardener-extensions/pkg/webhook" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" @@ -27,15 +28,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" ) -var logger = log.Log.WithName("aws-controlplaneexposure-webhook") +var ( + // DefaultAddOptions are the default AddOptions for AddToManager. + DefaultAddOptions = AddOptions{} +) -// AddToManager adds a webhook to the given manager. -func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { +// AddOptions are options to apply when adding the AWS exposure webhook to the manager. +type AddOptions struct { + // ETCDStorage is the etcd storage configuration. + ETCDStorage config.ETCDStorage +} + +var logger = log.Log.WithName("gcp-controlplaneexposure-webhook") + +// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager. +func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (webhook.Webhook, error) { logger.Info("Adding webhook to manager") return controlplane.Add(mgr, controlplane.AddArgs{ Kind: extensionswebhook.SeedKind, Provider: gcp.Type, - Types: []runtime.Object{&appsv1.Deployment{}}, - Mutator: genericmutator.NewMutator(NewEnsurer(logger), nil, nil, logger), + Types: []runtime.Object{&appsv1.Deployment{}, &appsv1.StatefulSet{}}, + Mutator: genericmutator.NewMutator(NewEnsurer(&opts.ETCDStorage, logger), nil, nil, logger), }) } + +// AddToManager creates a webhook with the default options and adds it to the manager. +func AddToManager(mgr manager.Manager) (webhook.Webhook, error) { + return AddToManagerWithOptions(mgr, DefaultAddOptions) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer.go index 84d631a60..0741e1496 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer.go +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer.go @@ -17,6 +17,8 @@ package controlplaneexposure import ( "context" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane/genericmutator" @@ -24,20 +26,23 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) // NewEnsurer creates a new controlplaneexposure ensurer. -func NewEnsurer(logger logr.Logger) genericmutator.Ensurer { +func NewEnsurer(etcdStorage *config.ETCDStorage, logger logr.Logger) genericmutator.Ensurer { return &ensurer{ - logger: logger.WithName("ensurer"), + etcdStorage: etcdStorage, + logger: logger.WithName("gcp-controlplaneexposure-ensurer"), } } type ensurer struct { - controlplane.NoopEnsurer - client client.Client - logger logr.Logger + genericmutator.NoopEnsurer + etcdStorage *config.ETCDStorage + client client.Client + logger logr.Logger } // InjectClient injects the given client into the ensurer. @@ -59,3 +64,22 @@ func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, dep *appsv1 } return nil } + +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *ensurer) EnsureETCDStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, cluster *extensionscontroller.Cluster) error { + e.ensureVolumeClaimTemplates(&ss.Spec, ss.Name) + return nil +} + +func (e *ensurer) ensureVolumeClaimTemplates(spec *appsv1.StatefulSetSpec, name string) { + t := e.getVolumeClaimTemplate(name) + spec.VolumeClaimTemplates = controlplane.EnsurePVCWithName(spec.VolumeClaimTemplates, *t) +} + +func (e *ensurer) getVolumeClaimTemplate(name string) *corev1.PersistentVolumeClaim { + var etcdStorage config.ETCDStorage + if name == common.EtcdMainStatefulSetName { + etcdStorage = *e.etcdStorage + } + return controlplane.GetETCDVolumeClaimTemplate(name, etcdStorage.ClassName, etcdStorage.Capacity) +} diff --git a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer_test.go b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer_test.go index ee413ddb1..30ebc265a 100644 --- a/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer_test.go +++ b/controllers/provider-gcp/pkg/webhook/controlplaneexposure/ensurer_test.go @@ -18,7 +18,9 @@ import ( "context" "testing" + "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/config" mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener-extensions/pkg/util" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/gardener/gardener/pkg/operation/common" @@ -27,6 +29,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -42,8 +45,13 @@ func TestController(t *testing.T) { RunSpecs(t, "GCP Controlplane Exposure Webhook Suite") } -var _ = Describe("Mutator", func() { +var _ = Describe("Ensurer", func() { var ( + etcdStorage = &config.ETCDStorage{ + ClassName: util.StringPtr("gardener.cloud-fast"), + Capacity: util.QuantityPtr(resource.MustParse("25Gi")), + } + ctrl *gomock.Controller svcKey = client.ObjectKey{Namespace: namespace, Name: common.KubeAPIServerDeploymentName} @@ -91,7 +99,7 @@ var _ = Describe("Mutator", func() { client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) // Create ensurer - ensurer := NewEnsurer(logger) + ensurer := NewEnsurer(etcdStorage, logger) err := ensurer.(inject.Client).InjectClient(client) Expect(err).To(Not(HaveOccurred())) @@ -125,7 +133,7 @@ var _ = Describe("Mutator", func() { client.EXPECT().Get(context.TODO(), svcKey, &corev1.Service{}).DoAndReturn(clientGet(svc)) // Create ensurer - ensurer := NewEnsurer(logger) + ensurer := NewEnsurer(etcdStorage, logger) err := ensurer.(inject.Client).InjectClient(client) Expect(err).To(Not(HaveOccurred())) @@ -135,6 +143,102 @@ var _ = Describe("Mutator", func() { checkKubeAPIServerDeployment(dep) }) }) + + Describe("#EnsureETCDStatefulSet", func() { + It("should add or modify elements to etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should modify existing elements of etcd-main statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-main"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDMainStatefulSet(ss) + }) + + It("should add or modify elements to etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + + It("should modify existing elements of etcd-events statefulset", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "etcd-events"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(etcdStorage, logger) + + // Call EnsureETCDStatefulSet method and check the result + err := ensurer.EnsureETCDStatefulSet(context.TODO(), ss, nil) + Expect(err).To(Not(HaveOccurred())) + checkETCDEventsStatefulSet(ss) + }) + }) }) func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { @@ -144,6 +248,17 @@ func checkKubeAPIServerDeployment(dep *appsv1.Deployment) { Expect(c.Command).To(ContainElement("--advertise-address=1.2.3.4")) } +func checkETCDMainStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-main") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdMainStatefulSetName, util.StringPtr("gardener.cloud-fast"), + util.QuantityPtr(resource.MustParse("25Gi"))))) +} + +func checkETCDEventsStatefulSet(ss *appsv1.StatefulSet) { + pvc := controlplane.PVCWithName(ss.Spec.VolumeClaimTemplates, "etcd-events") + Expect(pvc).To(Equal(controlplane.GetETCDVolumeClaimTemplate(common.EtcdEventsStatefulSetName, nil, nil))) +} + func clientGet(result runtime.Object) interface{} { return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { switch obj.(type) { diff --git a/controllers/provider-packet/cmd/gardener-extension-provider-packet/app/app.go b/controllers/provider-packet/cmd/gardener-extension-provider-packet/app/app.go index d3497859e..ce9f79783 100644 --- a/controllers/provider-packet/cmd/gardener-extension-provider-packet/app/app.go +++ b/controllers/provider-packet/cmd/gardener-extension-provider-packet/app/app.go @@ -64,8 +64,18 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { MaxConcurrentReconciles: 5, } - controllerSwitches = packetcmd.ControllerSwitchOptions() - webhookSwitches = packetcmd.WebhookAddToManagerOptions() + controllerSwitches = packetcmd.ControllerSwitchOptions() + webhookSwitches = packetcmd.WebhookSwitchOptions() + webhookServerOptions = &webhookcmd.ServerOptions{ + Port: 7890, + CertDir: "/tmp/cert", + Mode: webhookcmd.ServiceMode, + Name: "webhooks", + Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), + ServiceSelectors: "{}", + Host: "localhost", + } + webhookOptions = webhookcmd.NewAddToManagerOptions("packet-webhooks", webhookServerOptions, webhookSwitches) aggOption = controllercmd.NewOptionAggregator( restOpts, @@ -74,20 +84,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.PrefixOption("controlplane-", controlPlaneCtrlOpts), controllercmd.PrefixOption("worker-", workerCtrlOpts), controllerSwitches, - webhookSwitches, + webhookOptions, ) ) - webhookSwitches.Server = webhookcmd.ServerOptions{ - Port: 7890, - CertDir: "/tmp/cert", - Mode: webhookcmd.ServiceMode, - Name: "webhooks", - Namespace: os.Getenv("WEBHOOK_CONFIG_NAMESPACE"), - ServiceSelectors: "{}", - Host: "localhost", - } - cmd := &cobra.Command{ Use: fmt.Sprintf("%s-controller-manager", packet.Name), @@ -123,7 +123,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { controllercmd.LogErrAndExit(err, "Could not add controllers to manager") } - if err := webhookSwitches.Completed().AddToManager(mgr); err != nil { + if err := webhookOptions.Completed().AddToManager(mgr); err != nil { controllercmd.LogErrAndExit(err, "Could not add webhooks to manager") } diff --git a/controllers/provider-packet/pkg/cmd/options.go b/controllers/provider-packet/pkg/cmd/options.go index 5f5040a1b..fa3afe35a 100644 --- a/controllers/provider-packet/pkg/cmd/options.go +++ b/controllers/provider-packet/pkg/cmd/options.go @@ -34,7 +34,7 @@ func ControllerSwitchOptions() *controllercmd.SwitchOptions { ) } -// WebhookAddToManagerOptions are the webhookcmd.AddToManagerOptions for the provider webhooks. -func WebhookAddToManagerOptions() *webhookcmd.AddToManagerOptions { - return webhookcmd.NewAddToManagerOptions("packet-webhooks") +// WebhookSwitchOptions are the webhookcmd.SwitchOptions for the provider webhooks. +func WebhookSwitchOptions() *webhookcmd.SwitchOptions { + return webhookcmd.NewSwitchOptions() } diff --git a/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go b/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go index 4bb05cca3..386d336a9 100644 --- a/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go +++ b/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator/mocks.go @@ -7,6 +7,7 @@ package genericmutator import ( context "context" unit "github.com/coreos/go-systemd/unit" + controller "github.com/gardener/gardener-extensions/pkg/controller" gomock "github.com/golang/mock/gomock" v1 "k8s.io/api/apps/v1" v10 "k8s.io/api/core/v1" @@ -37,6 +38,20 @@ func (m *MockEnsurer) EXPECT() *MockEnsurerMockRecorder { return m.recorder } +// EnsureETCDStatefulSet mocks base method +func (m *MockEnsurer) EnsureETCDStatefulSet(arg0 context.Context, arg1 *v1.StatefulSet, arg2 *controller.Cluster) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureETCDStatefulSet", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureETCDStatefulSet indicates an expected call of EnsureETCDStatefulSet +func (mr *MockEnsurerMockRecorder) EnsureETCDStatefulSet(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureETCDStatefulSet", reflect.TypeOf((*MockEnsurer)(nil).EnsureETCDStatefulSet), arg0, arg1, arg2) +} + // EnsureKubeAPIServerDeployment mocks base method func (m *MockEnsurer) EnsureKubeAPIServerDeployment(arg0 context.Context, arg1 *v1.Deployment) error { m.ctrl.T.Helper() diff --git a/pkg/util/pointers.go b/pkg/util/pointers.go index 77476fd3a..42c5e2a33 100644 --- a/pkg/util/pointers.go +++ b/pkg/util/pointers.go @@ -14,6 +14,8 @@ package util +import "k8s.io/apimachinery/pkg/api/resource" + // BoolPtr returns a bool pointer to its argument. func BoolPtr(b bool) *bool { return &b @@ -28,3 +30,8 @@ func Int32Ptr(i int32) *int32 { func StringPtr(s string) *string { return &s } + +// QuantityPtr returns a Quatity pointer to its argument +func QuantityPtr(q resource.Quantity) *resource.Quantity { + return &q +} diff --git a/pkg/webhook/cmd/options.go b/pkg/webhook/cmd/options.go index 198f8b258..c861c37a0 100644 --- a/pkg/webhook/cmd/options.go +++ b/pkg/webhook/cmd/options.go @@ -228,11 +228,12 @@ type AddToManagerOptions struct { Switch SwitchOptions } -// NewAddToManagerOptions creates new AddToManagerOptions with the given server name and initial switch pairs. -func NewAddToManagerOptions(serverName string, pairs ...NameToFactory) *AddToManagerOptions { +// NewAddToManagerOptions creates new AddToManagerOptions with the given server name, server, and switch options. +func NewAddToManagerOptions(serverName string, serverOpts *ServerOptions, switchOpts *SwitchOptions) *AddToManagerOptions { return &AddToManagerOptions{ serverName: serverName, - Switch: *NewSwitchOptions(pairs...), + Server: *serverOpts, + Switch: *switchOpts, } } diff --git a/pkg/webhook/controlplane/controlplane.go b/pkg/webhook/controlplane/controlplane.go index a4be100db..a82c34be5 100644 --- a/pkg/webhook/controlplane/controlplane.go +++ b/pkg/webhook/controlplane/controlplane.go @@ -29,6 +29,8 @@ const ( WebhookName = "controlplane" // ExposureWebhookName is the exposure webhook name. ExposureWebhookName = "controlplaneexposure" + // BackupWebhookName is the backup webhook name. + BackupWebhookName = "controlplanebackup" ) var logger = log.Log.WithName("controlplane-webhook") @@ -67,8 +69,12 @@ func Add(mgr manager.Manager, args AddArgs) (webhook.Webhook, error) { } func getName(kind extensionswebhook.Kind) string { - if kind == extensionswebhook.SeedKind { + switch kind { + case extensionswebhook.SeedKind: return ExposureWebhookName + case extensionswebhook.BackupKind: + return BackupWebhookName + default: + return WebhookName } - return WebhookName } diff --git a/pkg/webhook/controlplane/etcd.go b/pkg/webhook/controlplane/etcd.go new file mode 100644 index 000000000..3484af854 --- /dev/null +++ b/pkg/webhook/controlplane/etcd.go @@ -0,0 +1,128 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetBackupRestoreContainer returns an etcd backup-restore container with the given name, schedule, provider, image, +// and additional provider-specific command line args and env variables. +func GetBackupRestoreContainer( + name, schedule, provider, image string, + args map[string]string, + env []corev1.EnvVar, + volumeMounts []corev1.VolumeMount, +) *corev1.Container { + c := &corev1.Container{ + Name: "backup-restore", + Command: []string{ + "etcdbrctl", + "server", + fmt.Sprintf("--schedule=%s", schedule), + "--data-dir=/var/etcd/data/new.etcd", + fmt.Sprintf("--storage-provider=%s", provider), + fmt.Sprintf("--store-prefix=%s", name), + "--cert=/var/etcd/ssl/client/tls.crt", + "--key=/var/etcd/ssl/client/tls.key", + "--cacert=/var/etcd/ssl/ca/ca.crt", + "--insecure-transport=false", + "--insecure-skip-tls-verify=false", + fmt.Sprintf("--endpoints=https://%s-0:2379", name), + "--etcd-connection-timeout=300", + "--delta-snapshot-period-seconds=300", + "--delta-snapshot-memory-limit=104857600", // 100MB + "--garbage-collection-period-seconds=43200", + "--snapstore-temp-directory=/var/etcd/data/temp", + }, + Env: []corev1.EnvVar{}, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{ + { + Name: "server", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: name, + MountPath: "/var/etcd/data", + }, + { + Name: "ca-etcd", + MountPath: "/var/etcd/ssl/ca", + }, + { + Name: "etcd-client-tls", + MountPath: "/var/etcd/ssl/client", + }, + }, + } + + // Ensure additional command line args + for k, v := range args { + c.Command = EnsureStringWithPrefix(c.Command, fmt.Sprintf("--%s=", k), v) + } + + // Ensure additional env variables + for _, envVar := range env { + c.Env = EnsureEnvVarWithName(c.Env, envVar) + } + + // Ensure additional volume mounts + for _, volumeMount := range volumeMounts { + c.VolumeMounts = EnsureVolumeMountWithName(c.VolumeMounts, volumeMount) + } + + return c +} + +func GetETCDVolumeClaimTemplate(name string, storageClassName *string, storageCapacity *resource.Quantity) *corev1.PersistentVolumeClaim { + // Determine the storage capacity + // A non-default storage capacity is used only if it's configured + capacity := resource.MustParse("10Gi") + if storageCapacity != nil { + capacity = *storageCapacity + } + + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: storageClassName, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: capacity, + }, + }, + }, + } +} diff --git a/pkg/webhook/controlplane/genericmutator/mutator.go b/pkg/webhook/controlplane/genericmutator/mutator.go index 2c4fc7ba8..816898706 100644 --- a/pkg/webhook/controlplane/genericmutator/mutator.go +++ b/pkg/webhook/controlplane/genericmutator/mutator.go @@ -17,6 +17,7 @@ package genericmutator import ( "context" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" "github.com/gardener/gardener-extensions/pkg/webhook/controlplane" "github.com/coreos/go-systemd/unit" @@ -43,6 +44,8 @@ type Ensurer interface { EnsureKubeControllerManagerDeployment(context.Context, *appsv1.Deployment) error // EnsureKubeSchedulerDeployment ensures that the kube-scheduler deployment conforms to the provider requirements. EnsureKubeSchedulerDeployment(context.Context, *appsv1.Deployment) error + // EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. + EnsureETCDStatefulSet(context.Context, *appsv1.StatefulSet, *extensionscontroller.Cluster) error // EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. EnsureKubeletServiceUnitOptions(context.Context, []*unit.UnitOption) ([]*unit.UnitOption, error) // EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. @@ -60,6 +63,7 @@ func NewMutator(ensurer Ensurer, unitSerializer controlplane.UnitSerializer, kub } type mutator struct { + client client.Client ensurer Ensurer unitSerializer controlplane.UnitSerializer kubeletConfigCodec controlplane.KubeletConfigCodec @@ -69,6 +73,7 @@ type mutator struct { // InjectClient injects the given client into the ensurer. // TODO Replace this with the more generic InjectFunc when controller runtime supports it func (m *mutator) InjectClient(client client.Client) error { + m.client = client if _, err := inject.ClientInto(client, m.ensurer); err != nil { return errors.Wrap(err, "could not inject the client into the ensurer") } @@ -92,6 +97,17 @@ func (m *mutator) Mutate(ctx context.Context, obj runtime.Object) error { case common.KubeSchedulerDeploymentName: return m.ensurer.EnsureKubeSchedulerDeployment(ctx, x) } + case *appsv1.StatefulSet: + switch x.Name { + case common.EtcdMainStatefulSetName, common.EtcdEventsStatefulSetName: + // Get cluster info + cluster, err := extensionscontroller.GetCluster(ctx, m.client, x.Namespace) + if err != nil { + return errors.Wrapf(err, "could not get cluster for namespace '%s'", x.Namespace) + } + + return m.ensurer.EnsureETCDStatefulSet(ctx, x, cluster) + } case *extensionsv1alpha1.OperatingSystemConfig: if x.Spec.Purpose == extensionsv1alpha1.OperatingSystemConfigPurposeReconcile { return m.mutateOperatingSystemConfig(ctx, x) diff --git a/pkg/webhook/controlplane/genericmutator/mutator_test.go b/pkg/webhook/controlplane/genericmutator/mutator_test.go index ca50c4607..edfe40b54 100644 --- a/pkg/webhook/controlplane/genericmutator/mutator_test.go +++ b/pkg/webhook/controlplane/genericmutator/mutator_test.go @@ -16,8 +16,11 @@ package genericmutator import ( "context" + "encoding/json" "testing" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + mockclient "github.com/gardener/gardener-extensions/pkg/mock/controller-runtime/client" mockcontrolplane "github.com/gardener/gardener-extensions/pkg/mock/gardener-extensions/webhook/controlplane" mockgenericmutator "github.com/gardener/gardener-extensions/pkg/mock/gardener-extensions/webhook/controlplane/genericmutator" "github.com/gardener/gardener-extensions/pkg/util" @@ -25,6 +28,7 @@ import ( "github.com/coreos/go-systemd/unit" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/garden/v1beta1" "github.com/gardener/gardener/pkg/operation/common" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" @@ -32,7 +36,10 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -44,6 +51,10 @@ const ( newKubeletConfigData = "new kubelet config data" ) +const ( + namespace = "test" +) + func TestControlplane(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Controlplane Webhook Generic Mutator Suite") @@ -53,6 +64,19 @@ var _ = Describe("Mutator", func() { var ( ctrl *gomock.Controller logger = log.Log.WithName("test") + + clusterKey = client.ObjectKey{Name: namespace} + cluster = &extensionscontroller.Cluster{ + CloudProfile: &gardenv1beta1.CloudProfile{}, + Seed: &gardenv1beta1.Seed{}, + Shoot: &gardenv1beta1.Shoot{ + Spec: gardenv1beta1.ShootSpec{ + Kubernetes: gardenv1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } ) BeforeEach(func() { @@ -170,6 +194,71 @@ var _ = Describe("Mutator", func() { Expect(err).To(Not(HaveOccurred())) }) + It("should invoke ensurer.EnsureETCDStatefulSet with a etcd-main stateful set", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdMainStatefulSetName, Namespace: namespace}, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), clusterKey, &extensionsv1alpha1.Cluster{}).DoAndReturn(clientGet(clusterObject(cluster))) + + // Create mock ensurer + ensurer := mockgenericmutator.NewMockEnsurer(ctrl) + ensurer.EXPECT().EnsureETCDStatefulSet(context.TODO(), ss, cluster).Return(nil) + + // Create mutator + mutator := NewMutator(ensurer, nil, nil, logger) + err := mutator.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call Mutate method and check the result + err = mutator.Mutate(context.TODO(), ss) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should invoke ensurer.EnsureETCDStatefulSet with a etcd-events stateful set", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.EtcdEventsStatefulSetName, Namespace: namespace}, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), clusterKey, &extensionsv1alpha1.Cluster{}).DoAndReturn(clientGet(clusterObject(cluster))) + + // Create mock ensurer + ensurer := mockgenericmutator.NewMockEnsurer(ctrl) + ensurer.EXPECT().EnsureETCDStatefulSet(context.TODO(), ss, cluster).Return(nil) + + // Create mutator + mutator := NewMutator(ensurer, nil, nil, logger) + err := mutator.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call Mutate method and check the result + err = mutator.Mutate(context.TODO(), ss) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should ignore other stateful sets than etcd-main and etcd-events", func() { + var ( + ss = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + } + ) + + // Create mutator + mutator := NewMutator(nil, nil, nil, logger) + + // Call Mutate method and check the result + err := mutator.Mutate(context.TODO(), ss) + Expect(err).To(Not(HaveOccurred())) + }) + It("should invoke appropriate ensurer methods with OperatingSystemConfig", func() { var ( osc = &extensionsv1alpha1.OperatingSystemConfig{ @@ -262,3 +351,34 @@ func checkOperatingSystemConfig(osc *extensionsv1alpha1.OperatingSystemConfig) { Expect(f).To(Not(BeNil())) Expect(f.Content.Inline).To(Equal(&extensionsv1alpha1.FileContentInline{Data: newKubeletConfigData})) } + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *extensionsv1alpha1.Cluster: + *obj.(*extensionsv1alpha1.Cluster) = *result.(*extensionsv1alpha1.Cluster) + } + return nil + } +} + +func clusterObject(cluster *extensionscontroller.Cluster) *extensionsv1alpha1.Cluster { + return &extensionsv1alpha1.Cluster{ + Spec: extensionsv1alpha1.ClusterSpec{ + CloudProfile: runtime.RawExtension{ + Raw: encode(cluster.CloudProfile), + }, + Seed: runtime.RawExtension{ + Raw: encode(cluster.Seed), + }, + Shoot: runtime.RawExtension{ + Raw: encode(cluster.Shoot), + }, + }, + } +} + +func encode(obj runtime.Object) []byte { + data, _ := json.Marshal(obj) + return data +} diff --git a/pkg/webhook/controlplane/noopensurer.go b/pkg/webhook/controlplane/genericmutator/noopensurer.go similarity index 88% rename from pkg/webhook/controlplane/noopensurer.go rename to pkg/webhook/controlplane/genericmutator/noopensurer.go index d80890322..3b6c391cb 100644 --- a/pkg/webhook/controlplane/noopensurer.go +++ b/pkg/webhook/controlplane/genericmutator/noopensurer.go @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package controlplane +package genericmutator import ( "context" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "github.com/coreos/go-systemd/unit" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -46,6 +48,11 @@ func (e *NoopEnsurer) EnsureKubeSchedulerDeployment(context.Context, *appsv1.Dep return nil } +// EnsureETCDStatefulSet ensures that the etcd stateful sets conform to the provider requirements. +func (e *NoopEnsurer) EnsureETCDStatefulSet(context.Context, *appsv1.StatefulSet, *extensionscontroller.Cluster) error { + return nil +} + // EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. func (e *NoopEnsurer) EnsureKubeletServiceUnitOptions(_ context.Context, opts []*unit.UnitOption) ([]*unit.UnitOption, error) { return opts, nil diff --git a/pkg/webhook/controlplane/utils.go b/pkg/webhook/controlplane/utils.go index 5cff5fa54..4049374bc 100644 --- a/pkg/webhook/controlplane/utils.go +++ b/pkg/webhook/controlplane/utils.go @@ -50,6 +50,14 @@ func ContainerWithName(containers []corev1.Container, name string) *corev1.Conta return nil } +// PVCWithName returns the PersistentVolumeClaim with the given name if it exists in the given slice, nil otherwise. +func PVCWithName(pvcs []corev1.PersistentVolumeClaim, name string) *corev1.PersistentVolumeClaim { + if i := pvcWithNameIndex(pvcs, name); i >= 0 { + return &pvcs[i] + } + return nil +} + // UnitWithName returns the unit with the given name if it exists in the given slice, nil otherwise. func UnitWithName(units []extensionsv1alpha1.Unit, name string) *extensionsv1alpha1.Unit { if i := unitWithNameIndex(units, name); i >= 0 { @@ -179,6 +187,44 @@ func EnsureNoVolumeWithName(items []corev1.Volume, name string) []corev1.Volume return items } +// EnsureContainerWithName ensures that a Container with a name equal to the name of the given Container exists +// in the given slice and is equal to the given Container. +func EnsureContainerWithName(items []corev1.Container, item corev1.Container) []corev1.Container { + if i := containerWithNameIndex(items, item.Name); i < 0 { + items = append(items, item) + } else if !reflect.DeepEqual(items[i], item) { + items = append(append(items[:i], item), items[i+1:]...) + } + return items +} + +// EnsureNoContainerWithName ensures that a Container with the given name does not exist in the given slice. +func EnsureNoContainerWithName(items []corev1.Container, name string) []corev1.Container { + if i := containerWithNameIndex(items, name); i >= 0 { + items = append(items[:i], items[i+1:]...) + } + return items +} + +// EnsurePVCWithName ensures that a PVC with a name equal to the name of the given PVC exists +// in the given slice and is equal to the given PVC. +func EnsurePVCWithName(items []corev1.PersistentVolumeClaim, item corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { + if i := pvcWithNameIndex(items, item.Name); i < 0 { + items = append(items, item) + } else if !reflect.DeepEqual(items[i], item) { + items = append(append(items[:i], item), items[i+1:]...) + } + return items +} + +// EnsureNoPVCWithName ensures that a PVC with the given name does not exist in the given slice. +func EnsureNoPVCWithName(items []corev1.PersistentVolumeClaim, name string) []corev1.PersistentVolumeClaim { + if i := pvcWithNameIndex(items, name); i >= 0 { + items = append(items[:i], items[i+1:]...) + } + return items +} + // EnsureUnitOption ensures the given unit option exist in the given slice. func EnsureUnitOption(items []*unit.UnitOption, item *unit.UnitOption) []*unit.UnitOption { if i := unitOptionIndex(items, item); i < 0 { @@ -278,3 +324,12 @@ func volumeWithNameIndex(items []corev1.Volume, name string) int { } return -1 } + +func pvcWithNameIndex(items []corev1.PersistentVolumeClaim, name string) int { + for i, item := range items { + if item.Name == name { + return i + } + } + return -1 +} diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index a98301fa6..ee731010e 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -36,6 +36,10 @@ const ( // ShootProviderLabel is a label on shoot namespaces in the seed cluster that identifies the Shoot provider. // TODO Move this constant to gardener/gardener ShootProviderLabel = "shoot.gardener.cloud/provider" + // BackupProviderLabel is a label on shoot namespaces in the seed cluster that identifies the Backup provider. + // This provider can be different from both the Seed or the Shoot provider, see https://github.com/gardener/gardener/blob/master/docs/proposals/02-backupinfra.md. + // TODO Move this constant to gardener/gardener + BackupProviderLabel = "backup.gardener.cloud/provider" ) // Kind is a type for webhook kinds. @@ -47,6 +51,8 @@ const ( SeedKind Kind = "seed" // A shoot webhook is applied only to those shoot namespaces that have the correct Shoot provider label. ShootKind Kind = "shoot" + // A backup webhook is applied only to those shoot namespaces that have the correct Backup provider label. + BackupKind Kind = "backup" ) // FactoryAggregator aggregates various Factory functions. @@ -113,7 +119,7 @@ func (s *ServerBuilder) AddToManager(mgr manager.Manager) error { return errors.Wrapf(err, "could not register webhooks in server %s", s.Name) } - return errors.Wrapf(mgr.Add(srv), "could not add webhook server %s to manager", s.Name) + return nil } // NewWebhook creates a new mutating webhook for create and update operations @@ -158,6 +164,8 @@ func buildSelector(kind Kind, provider string) (*metav1.LabelSelector, error) { key = SeedProviderLabel case ShootKind: key = ShootProviderLabel + case BackupKind: + key = BackupProviderLabel default: return nil, errors.Errorf("invalid webhook kind '%s'", kind) }