Skip to content

Commit

Permalink
move providerID flag to configuration file of kubelet (#460)
Browse files Browse the repository at this point in the history 
* move providerid flag to configuration file of kubelet

* add unit test

* correct script

* correct test case
  • Loading branch information
@shaoyongfeng
shaoyongfeng authored Mar 14, 2022
1 parent 0719f8f commit 11c81b3
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 26 deletions.
48 changes: 32 additions & 16 deletions pkg/webhook/controlplane/ensurer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,37 +99,52 @@ func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, gctx gcon
command = ensureKubeletCommandLineArgs(command, kubeletVersion)
opt.Value = extensionswebhook.SerializeCommandLine(command, 1, " \\\n ")
}

/*
* # Set environment PROVIDER_ID to /var/lib/kubelet/extra_args
*
* grep -sq PROVIDER_ID /var/lib/kubelet/extra_args
* if [ $? -ne 0 ]; then
* META_EP=http://100.100.100.200/latest/meta-data
* PROVIDER_ID=`wget -qO- $META_EP/region-id $META_EP/region-id`.`wget -qO- $META_EP/region-id $META_EP/instance-id`
* echo PROVIDER_ID=$PROVIDER_ID >> /var/lib/kubelet/extra_args
* echo PROVIDER_ID=$PROVIDER_ID has been written to /var/lib/kubelet/extra_args
* fi
*/
new = extensionswebhook.EnsureUnitOption(new, &unit.UnitOption{
Section: "Service",
Name: "ExecStartPre",
//This doesn't work: /bin/sh -c "$(echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL2V0Yy9lbnZpcm9ubWVudAppZiBbICQ.... | base64 -d)"
Value: `/bin/sh -c "echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmlmIFsgJD8gLW5lIDAgXTsgdGhlbgpNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQpQUk9WSURFUl9JRD1gd2dldCAtcU8tICRNRVRBX0VQL3JlZ2lvbi1pZGAuYHdnZXQgLXFPLSAkTUVUQV9FUC9pbnN0YW5jZS1pZGAKZWNobyBQUk9WSURFUl9JRD0kUFJPVklERVJfSUQgPj4gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmVjaG8gUFJPVklERVJfSUQ9JFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmZpCg==| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`,
Value: getValueOfKubeletPreStart(kubeletVersion),
})
return new, nil
}

func getValueOfKubeletPreStart(kubeletVersion *semver.Version) string {
if version.ConstraintK8sGreaterEqual123.Check(kubeletVersion) {
/*
# set providerid in /var/lib/kubelet/config/kubelet
grep -sq place_holder_of_providerid /var/lib/kubelet/config/kubelet
if [ $? -eq 0 ]; then
META_EP=http://100.100.100.200/latest/meta-data
PROVIDER_ID=`wget -qO- $META_EP/region-id`.`wget -qO- $META_EP/instance-id`
sudo sed -i "s/place_holder_of_providerid/${PROVIDER_ID}/g" /var/lib/kubelet/config/kubelet
echo "providerID= $PROVIDER_ID has been written to /var/lib/kubelet/config/kubelet"
fi
*/
return `/bin/sh -c "echo IyBzZXQgcHJvdmlkZXJpZCBpbiAvdmFyL2xpYi9rdWJlbGV0L2NvbmZpZy9rdWJlbGV0CmdyZXAgLXNxIHBsYWNlX2hvbGRlcl9vZl9wcm92aWRlcmlkIC92YXIvbGliL2t1YmVsZXQvY29uZmlnL2t1YmVsZXQKaWYgWyAkPyAtZXEgMCBdOyB0aGVuCiAgICBNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQogICAgUFJPVklERVJfSUQ9YHdnZXQgLXFPLSAkTUVUQV9FUC9yZWdpb24taWRgLmB3Z2V0IC1xTy0gJE1FVEFfRVAvaW5zdGFuY2UtaWRgCiAgICBzdWRvIHNlZCAgLWkgInMvcGxhY2VfaG9sZGVyX29mX3Byb3ZpZGVyaWQvJHtQUk9WSURFUl9JRH0vZyIgL3Zhci9saWIva3ViZWxldC9jb25maWcva3ViZWxldAogICAgZWNobyAicHJvdmlkZXJJRD0gJFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9jb25maWcva3ViZWxldCIKZmkK| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`
} else {
/*
* # Set environment PROVIDER_ID to /var/lib/kubelet/extra_args
*
* grep -sq PROVIDER_ID /var/lib/kubelet/extra_args
* if [ $? -ne 0 ]; then
* META_EP=http://100.100.100.200/latest/meta-data
* PROVIDER_ID=`wget -qO- $META_EP/region-id`.`wget -qO- $META_EP/instance-id`
* echo PROVIDER_ID=$PROVIDER_ID >> /var/lib/kubelet/extra_args
* echo PROVIDER_ID=$PROVIDER_ID has been written to /var/lib/kubelet/extra_args
* fi
*/
//This doesn't work: /bin/sh -c "$(echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL2V0Yy9lbnZpcm9ubWVudAppZiBbICQ.... | base64 -d)"
return `/bin/sh -c "echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmlmIFsgJD8gLW5lIDAgXTsgdGhlbgpNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQpQUk9WSURFUl9JRD1gd2dldCAtcU8tICRNRVRBX0VQL3JlZ2lvbi1pZGAuYHdnZXQgLXFPLSAkTUVUQV9FUC9pbnN0YW5jZS1pZGAKZWNobyBQUk9WSURFUl9JRD0kUFJPVklERVJfSUQgPj4gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmVjaG8gUFJPVklERVJfSUQ9JFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmZpCg==| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`
}
}
func ensureKubeletCommandLineArgs(command []string, kubeletVersion *semver.Version) []string {
// TODO: Figure out how to provide the provider-id via the kubelet config file (as of Kubernetes 1.19 the kubelet config
// offers a new `providerID` field which can be used, and it's expected that `--provider-id` will be deprecated eventually).
// Today, the problem is that the provider ID is determined dynamically using the script above, but the kubelet config cannot
// reference environment variables like it's possible today with the CLI parameters.
// See https://github.com/kubernetes/kubernetes/pull/90494
command = extensionswebhook.EnsureStringWithPrefix(command, "--provider-id=", "${PROVIDER_ID}")
command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-provider=", "external")

if !version.ConstraintK8sGreaterEqual123.Check(kubeletVersion) {
command = extensionswebhook.EnsureStringWithPrefix(command, "--provider-id=", "${PROVIDER_ID}")
command = extensionswebhook.EnsureStringWithPrefix(command, "--enable-controller-attach-detach=", "true")
}

Expand All @@ -140,6 +155,7 @@ func ensureKubeletCommandLineArgs(command []string, kubeletVersion *semver.Versi
func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, gctx gcontext.GardenContext, kubeletVersion *semver.Version, new, old *kubeletconfigv1beta1.KubeletConfiguration) error {
if version.ConstraintK8sGreaterEqual123.Check(kubeletVersion) {
new.EnableControllerAttachDetach = pointer.Bool(true)
new.ProviderID = "place_holder_of_providerid"
}

return nil
Expand Down
34 changes: 24 additions & 10 deletions pkg/webhook/controlplane/ensurer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,19 +213,22 @@ var _ = Describe("Ensurer", func() {
})

DescribeTable("should modify existing elements of kubelet.service unit options",
func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, cloudProvider string, withControllerAttachDetachFlag bool) {
func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, cloudProvider string, equalGreaterV123 bool) {
valueofPreStart := `/bin/sh -c "echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmlmIFsgJD8gLW5lIDAgXTsgdGhlbgpNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQpQUk9WSURFUl9JRD1gd2dldCAtcU8tICRNRVRBX0VQL3JlZ2lvbi1pZGAuYHdnZXQgLXFPLSAkTUVUQV9FUC9pbnN0YW5jZS1pZGAKZWNobyBQUk9WSURFUl9JRD0kUFJPVklERVJfSUQgPj4gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmVjaG8gUFJPVklERVJfSUQ9JFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmZpCg==| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`
if equalGreaterV123 {
valueofPreStart = `/bin/sh -c "echo IyBzZXQgcHJvdmlkZXJpZCBpbiAvdmFyL2xpYi9rdWJlbGV0L2NvbmZpZy9rdWJlbGV0CmdyZXAgLXNxIHBsYWNlX2hvbGRlcl9vZl9wcm92aWRlcmlkIC92YXIvbGliL2t1YmVsZXQvY29uZmlnL2t1YmVsZXQKaWYgWyAkPyAtZXEgMCBdOyB0aGVuCiAgICBNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQogICAgUFJPVklERVJfSUQ9YHdnZXQgLXFPLSAkTUVUQV9FUC9yZWdpb24taWRgLmB3Z2V0IC1xTy0gJE1FVEFfRVAvaW5zdGFuY2UtaWRgCiAgICBzdWRvIHNlZCAgLWkgInMvcGxhY2VfaG9sZGVyX29mX3Byb3ZpZGVyaWQvJHtQUk9WSURFUl9JRH0vZyIgL3Zhci9saWIva3ViZWxldC9jb25maWcva3ViZWxldAogICAgZWNobyAicHJvdmlkZXJJRD0gJFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9jb25maWcva3ViZWxldCIKZmkK| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`
}
newUnitOptions := []*unit.UnitOption{
{
Section: "Service",
Name: "ExecStart",
Value: `/opt/bin/hyperkube kubelet \
--config=/var/lib/kubelet/config/kubelet \
--provider-id=${PROVIDER_ID}`,
--config=/var/lib/kubelet/config/kubelet`,
},
{
Section: "Service",
Name: "ExecStartPre",
Value: `/bin/sh -c "echo Z3JlcCAtc3EgUFJPVklERVJfSUQgL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmlmIFsgJD8gLW5lIDAgXTsgdGhlbgpNRVRBX0VQPWh0dHA6Ly8xMDAuMTAwLjEwMC4yMDAvbGF0ZXN0L21ldGEtZGF0YQpQUk9WSURFUl9JRD1gd2dldCAtcU8tICRNRVRBX0VQL3JlZ2lvbi1pZGAuYHdnZXQgLXFPLSAkTUVUQV9FUC9pbnN0YW5jZS1pZGAKZWNobyBQUk9WSURFUl9JRD0kUFJPVklERVJfSUQgPj4gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmVjaG8gUFJPVklERVJfSUQ9JFBST1ZJREVSX0lEIGhhcyBiZWVuIHdyaXR0ZW4gdG8gL3Zhci9saWIva3ViZWxldC9leHRyYV9hcmdzCmZpCg==| base64 -d > /var/lib/kubelet/gardener-set-provider-id && chmod +x /var/lib/kubelet/gardener-set-provider-id && /var/lib/kubelet/gardener-set-provider-id"`,
Value: valueofPreStart,
},
}

Expand All @@ -234,7 +237,9 @@ var _ = Describe("Ensurer", func() {
--cloud-provider=` + cloudProvider
}

if withControllerAttachDetachFlag {
if !equalGreaterV123 {
newUnitOptions[0].Value += ` \
--provider-id=${PROVIDER_ID}`
newUnitOptions[0].Value += ` \
--enable-controller-attach-detach=true`
}
Expand All @@ -244,8 +249,8 @@ var _ = Describe("Ensurer", func() {
Expect(opts).To(Equal(newUnitOptions))
},

Entry("kubelet version < 1.23", eContext18, semver.MustParse("1.20.0"), "external", true),
Entry("kubelet version >= 1.23", eContext18, semver.MustParse("1.23.0"), "external", false),
Entry("kubelet version < 1.23", eContext18, semver.MustParse("1.20.0"), "external", false),
Entry("kubelet version >= 1.23", eContext18, semver.MustParse("1.23.0"), "external", true),
)
})

Expand All @@ -265,12 +270,21 @@ var _ = Describe("Ensurer", func() {
})

DescribeTable("should modify existing elements of kubelet configuration",
func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, csiFeatureGateName string, enableControllerAttachDetach *bool) {
func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, csiFeatureGateName string, equalGreaterV123 bool) {
var (
enableControllerAttachDetach *bool = nil
providerId string = ""
)
if equalGreaterV123 {
enableControllerAttachDetach = pointer.Bool(true)
providerId = "place_holder_of_providerid"
}
newKubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{
FeatureGates: map[string]bool{
"Foo": true,
},
EnableControllerAttachDetach: enableControllerAttachDetach,
ProviderID: providerId,
}

if csiFeatureGateName != "" {
Expand All @@ -284,8 +298,8 @@ var _ = Describe("Ensurer", func() {
Expect(&kubeletConfig).To(Equal(newKubeletConfig))
},

Entry("1.18 <= kubelet < 1.23", eContext18, semver.MustParse("1.20.0"), "", nil),
Entry("kubelet >= 1.23", eContext18, semver.MustParse("1.23.0"), "", pointer.Bool(true)),
Entry("1.18 <= kubelet < 1.23", eContext18, semver.MustParse("1.20.0"), "", false),
Entry("kubelet >= 1.23", eContext18, semver.MustParse("1.23.0"), "", true),
)
})
})
Expand Down

0 comments on commit 11c81b3

Please sign in to comment.