Skip to content

Commit 9fdcb5f

Browse files
sushrkorsenthil
andauthored
disable leaked eni cleanup routine when vpc-resource-controller is deployed (#2854)
* disable leaked ENI cleanup routine when vpc-resource-controller is deployed * update helm version --------- Co-authored-by: Senthil Kumaran <senthilx@amazon.com>
1 parent 83b8704 commit 9fdcb5f

File tree

13 files changed

+453
-91
lines changed

13 files changed

+453
-91
lines changed

go.mod

+3-3
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ require (
2424
github.com/spf13/pflag v1.0.5
2525
github.com/stretchr/testify v1.9.0
2626
github.com/vishvananda/netlink v1.2.1-beta.2
27-
go.uber.org/zap v1.27.0
27+
go.uber.org/zap v1.26.0
2828
golang.org/x/net v0.24.0
2929
golang.org/x/sys v0.19.0
3030
google.golang.org/grpc v1.62.0
3131
google.golang.org/protobuf v1.33.0
3232
gopkg.in/natefinch/lumberjack.v2 v2.2.1
3333
gopkg.in/yaml.v2 v2.4.0
3434
helm.sh/helm/v3 v3.14.3
35-
k8s.io/api v0.30.1
36-
k8s.io/apimachinery v0.30.1
35+
k8s.io/api v0.29.3
36+
k8s.io/apimachinery v0.29.3
3737
k8s.io/cli-runtime v0.29.0
3838
k8s.io/client-go v0.29.3
3939
sigs.k8s.io/controller-runtime v0.17.0

go.sum

+6-6
Original file line numberDiff line numberDiff line change
@@ -460,8 +460,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
460460
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
461461
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
462462
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
463-
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
464-
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
463+
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
464+
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
465465
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
466466
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
467467
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -608,12 +608,12 @@ helm.sh/helm/v3 v3.14.3 h1:HmvRJlwyyt9HjgmAuxHbHv3PhMz9ir/XNWHyXfmnOP4=
608608
helm.sh/helm/v3 v3.14.3/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
609609
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
610610
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
611-
k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
612-
k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
611+
k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
612+
k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
613613
k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0=
614614
k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc=
615-
k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
616-
k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
615+
k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
616+
k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
617617
k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o=
618618
k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM=
619619
k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4=

pkg/awsutils/awsutils.go

+30-15
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore"
3131

3232
"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/awssession"
33+
"github.com/aws/amazon-vpc-cni-k8s/pkg/config"
3334
"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper"
3435
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder"
3536
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
@@ -54,11 +55,11 @@ const (
5455

5556
// AllocENI need to choose a first free device number between 0 and maxENI
5657
// 100 is a hard limit because we use vlanID + 100 for pod networking table names
57-
maxENIs = 100
58-
clusterNameEnvVar = "CLUSTER_NAME"
59-
eniNodeTagKey = "node.k8s.amazonaws.com/instance_id"
60-
eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt"
61-
eniClusterTagKey = "cluster.k8s.amazonaws.com/name"
58+
maxENIs = 100
59+
60+
// ENI tags
61+
eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt"
62+
6263
additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS"
6364
reservedTagKeyPrefix = "k8s.amazonaws.com"
6465
subnetDiscoveryTagKey = "kubernetes.io/role/cni"
@@ -213,6 +214,8 @@ type EC2InstanceMetadataCache struct {
213214
enablePrefixDelegation bool
214215

215216
clusterName string
217+
clusterNameEnvVal string
218+
nodeName string
216219
additionalENITags map[string]string
217220

218221
imds TypedIMDS
@@ -353,15 +356,17 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string)
353356
}
354357

355358
// New creates an EC2InstanceMetadataCache
356-
func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool) (*EC2InstanceMetadataCache, error) {
359+
func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool, clusterName, nodeName string) (*EC2InstanceMetadataCache, error) {
357360
// ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run
358361
ctx := context.Background()
359362

360363
sess := awssession.New()
361364
ec2Metadata := ec2metadata.New(sess)
362365
cache := &EC2InstanceMetadataCache{}
363366
cache.imds = TypedIMDS{instrumentedIMDS{ec2Metadata}}
364-
cache.clusterName = os.Getenv(clusterNameEnvVar)
367+
cache.clusterName = clusterName
368+
cache.clusterNameEnvVal = os.Getenv(config.ClusterNameEnv)
369+
cache.nodeName = nodeName
365370
cache.additionalENITags = loadAdditionalENITags()
366371

367372
region, err := ec2Metadata.Region()
@@ -982,14 +987,24 @@ func (cache *EC2InstanceMetadataCache) tryCreateNetworkInterface(input *ec2.Crea
982987
// buildENITags computes the desired AWS Tags for eni
983988
func (cache *EC2InstanceMetadataCache) buildENITags() map[string]string {
984989
tags := map[string]string{
985-
eniNodeTagKey: cache.instanceID,
990+
// TODO: deprecate instance ID tag to replace with nodename to align with tag used in vpc-resource-controller
991+
config.ENIInstanceIDTag: cache.instanceID,
986992
}
987993

988-
// If clusterName is provided,
989-
// tag the ENI with "cluster.k8s.amazonaws.com/name=<cluster_name>"
994+
// clusterName is set from CNINode created by vpc-resource-controller, add the new tags only when it is set so controller can deleted leaked ENIs
995+
// If it is not set then likely the controller is not running, so skip
990996
if cache.clusterName != "" {
991-
tags[eniClusterTagKey] = cache.clusterName
997+
tags[fmt.Sprintf(config.ClusterNameTagKeyFormat, cache.clusterName)] = config.ClusterNameTagValue
998+
tags[config.ENINodeNameTagKey] = cache.nodeName
999+
tags[config.ENIOwnerTagKey] = config.ENIOwnerTagValue
1000+
}
1001+
1002+
if cache.clusterNameEnvVal != "" {
1003+
// TODO: deprecate this tag to replace with "kubernetes.io/cluster/<cluster-name>:owned" to align with tag used in vpc-resource-controller
1004+
// for backward compatibily, add tag if CLUSTER_NAME ENV is set
1005+
tags[config.ClusterNameTagKey] = cache.clusterNameEnvVal
9921006
}
1007+
9931008
for key, value := range cache.additionalENITags {
9941009
tags[key] = value
9951010
}
@@ -1877,7 +1892,7 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface,
18771892
{
18781893
Name: aws.String("tag-key"),
18791894
Values: []*string{
1880-
aws.String(eniNodeTagKey),
1895+
aws.String(config.ENIInstanceIDTag),
18811896
},
18821897
},
18831898
{
@@ -1893,11 +1908,11 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface,
18931908
},
18941909
},
18951910
}
1896-
if cache.clusterName != "" {
1911+
if cache.clusterNameEnvVal != "" {
18971912
leakedENIFilters = append(leakedENIFilters, &ec2.Filter{
1898-
Name: aws.String(fmt.Sprintf("tag:%s", eniClusterTagKey)),
1913+
Name: aws.String(fmt.Sprintf("tag:%s", config.ClusterNameTagKey)),
18991914
Values: []*string{
1900-
aws.String(cache.clusterName),
1915+
aws.String(cache.clusterNameEnvVal),
19011916
},
19021917
})
19031918
}

0 commit comments

Comments
 (0)