From 9fdcb5f96c56154f5cfaaec2ea049e6c5bb14979 Mon Sep 17 00:00:00 2001 From: Sushmitha Ravikumar <58063229+sushrk@users.noreply.github.com> Date: Wed, 12 Jun 2024 18:19:53 -0700 Subject: [PATCH] disable leaked eni cleanup routine when vpc-resource-controller is deployed (#2854) * disable leaked ENI cleanup routine when vpc-resource-controller is deployed * update helm version --------- Co-authored-by: Senthil Kumaran --- go.mod | 6 +- go.sum | 12 +- pkg/awsutils/awsutils.go | 45 +++-- pkg/awsutils/awsutils_test.go | 190 ++++++++++++++---- pkg/config/type.go | 32 +++ pkg/ipamd/ipamd.go | 44 +++- pkg/ipamd/ipamd_test.go | 108 ++++++++-- test/framework/resources/k8s/manager.go | 7 + .../resources/k8s/resources/cninode.go | 41 ++++ test/integration/ipamd/cninode_test.go | 36 ++++ test/integration/ipamd/eni_ip_leak_test.go | 3 - test/integration/ipamd/eni_tag_test.go | 18 ++ test/integration/ipamd/ipamd_suite_test.go | 2 +- 13 files changed, 453 insertions(+), 91 deletions(-) create mode 100644 pkg/config/type.go create mode 100644 test/framework/resources/k8s/resources/cninode.go create mode 100644 test/integration/ipamd/cninode_test.go diff --git a/go.mod b/go.mod index cba660a484..fdc96a6f54 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/vishvananda/netlink v1.2.1-beta.2 - go.uber.org/zap v1.27.0 + go.uber.org/zap v1.26.0 golang.org/x/net v0.24.0 golang.org/x/sys v0.19.0 google.golang.org/grpc v1.62.0 @@ -32,8 +32,8 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.14.3 - k8s.io/api v0.30.1 - k8s.io/apimachinery v0.30.1 + k8s.io/api v0.29.3 + k8s.io/apimachinery v0.29.3 k8s.io/cli-runtime v0.29.0 k8s.io/client-go v0.29.3 sigs.k8s.io/controller-runtime v0.17.0 diff --git a/go.sum b/go.sum index 05ccee6cfa..1632fc53e1 100644 --- a/go.sum +++ b/go.sum @@ -460,8 +460,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -608,12 +608,12 @@ helm.sh/helm/v3 v3.14.3 h1:HmvRJlwyyt9HjgmAuxHbHv3PhMz9ir/XNWHyXfmnOP4= helm.sh/helm/v3 v3.14.3/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= -k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= -k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= -k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o= k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM= k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4= diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index f9ba346915..c88bac4e4d 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -30,6 +30,7 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/awssession" + "github.com/aws/amazon-vpc-cni-k8s/pkg/config" "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" @@ -54,11 +55,11 @@ const ( // AllocENI need to choose a first free device number between 0 and maxENI // 100 is a hard limit because we use vlanID + 100 for pod networking table names - maxENIs = 100 - clusterNameEnvVar = "CLUSTER_NAME" - eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" - eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt" - eniClusterTagKey = "cluster.k8s.amazonaws.com/name" + maxENIs = 100 + + // ENI tags + eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt" + additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS" reservedTagKeyPrefix = "k8s.amazonaws.com" subnetDiscoveryTagKey = "kubernetes.io/role/cni" @@ -213,6 +214,8 @@ type EC2InstanceMetadataCache struct { enablePrefixDelegation bool clusterName string + clusterNameEnvVal string + nodeName string additionalENITags map[string]string imds TypedIMDS @@ -353,7 +356,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) } // New creates an EC2InstanceMetadataCache -func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool) (*EC2InstanceMetadataCache, error) { +func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool, clusterName, nodeName string) (*EC2InstanceMetadataCache, error) { // ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run ctx := context.Background() @@ -361,7 +364,9 @@ func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Ena ec2Metadata := ec2metadata.New(sess) cache := &EC2InstanceMetadataCache{} cache.imds = TypedIMDS{instrumentedIMDS{ec2Metadata}} - cache.clusterName = os.Getenv(clusterNameEnvVar) + cache.clusterName = clusterName + cache.clusterNameEnvVal = os.Getenv(config.ClusterNameEnv) + cache.nodeName = nodeName cache.additionalENITags = loadAdditionalENITags() region, err := ec2Metadata.Region() @@ -982,14 +987,24 @@ func (cache *EC2InstanceMetadataCache) tryCreateNetworkInterface(input *ec2.Crea // buildENITags computes the desired AWS Tags for eni func (cache *EC2InstanceMetadataCache) buildENITags() map[string]string { tags := map[string]string{ - eniNodeTagKey: cache.instanceID, + // TODO: deprecate instance ID tag to replace with nodename to align with tag used in vpc-resource-controller + config.ENIInstanceIDTag: cache.instanceID, } - // If clusterName is provided, - // tag the ENI with "cluster.k8s.amazonaws.com/name=" + // clusterName is set from CNINode created by vpc-resource-controller, add the new tags only when it is set so controller can deleted leaked ENIs + // If it is not set then likely the controller is not running, so skip if cache.clusterName != "" { - tags[eniClusterTagKey] = cache.clusterName + tags[fmt.Sprintf(config.ClusterNameTagKeyFormat, cache.clusterName)] = config.ClusterNameTagValue + tags[config.ENINodeNameTagKey] = cache.nodeName + tags[config.ENIOwnerTagKey] = config.ENIOwnerTagValue + } + + if cache.clusterNameEnvVal != "" { + // TODO: deprecate this tag to replace with "kubernetes.io/cluster/:owned" to align with tag used in vpc-resource-controller + // for backward compatibily, add tag if CLUSTER_NAME ENV is set + tags[config.ClusterNameTagKey] = cache.clusterNameEnvVal } + for key, value := range cache.additionalENITags { tags[key] = value } @@ -1877,7 +1892,7 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface, { Name: aws.String("tag-key"), Values: []*string{ - aws.String(eniNodeTagKey), + aws.String(config.ENIInstanceIDTag), }, }, { @@ -1893,11 +1908,11 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface, }, }, } - if cache.clusterName != "" { + if cache.clusterNameEnvVal != "" { leakedENIFilters = append(leakedENIFilters, &ec2.Filter{ - Name: aws.String(fmt.Sprintf("tag:%s", eniClusterTagKey)), + Name: aws.String(fmt.Sprintf("tag:%s", config.ClusterNameTagKey)), Values: []*string{ - aws.String(cache.clusterName), + aws.String(cache.clusterNameEnvVal), }, }) } diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index cf93040526..fb84f3829f 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -31,6 +31,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/amazon-vpc-cni-k8s/pkg/config" mock_ec2wrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder" v1 "k8s.io/api/core/v1" @@ -1142,7 +1143,7 @@ func TestEC2InstanceMetadataCache_cleanUpLeakedENIsInternal(t *testing.T) { interfaces := []*ec2.NetworkInterface{{ Description: &description, TagSet: []*ec2.Tag{ - {Key: aws.String(eniNodeTagKey), Value: aws.String("test-value")}, + {Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("test-value")}, }, }} @@ -1170,7 +1171,9 @@ func setupDescribeNetworkInterfacesPagesWithContextMock( func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { type fields struct { instanceID string + nodeName string clusterName string + clusterNameEnv string additionalENITags map[string]string } tests := []struct { @@ -1182,35 +1185,52 @@ func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { name: "without clusterName or additionalENITags", fields: fields{ instanceID: "i-xxxxx", + nodeName: "fake-node", }, want: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxxx", + config.ENIInstanceIDTag: "i-xxxxx", }, }, { name: "with clusterName", fields: fields{ instanceID: "i-xxxxx", + nodeName: "fake-node", clusterName: "awesome-cluster", }, want: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxxx", - "cluster.k8s.amazonaws.com/name": "awesome-cluster", + config.ENIInstanceIDTag: "i-xxxxx", + config.ENINodeNameTagKey: "fake-node", + config.ENIOwnerTagKey: config.ENIOwnerTagValue, + fmt.Sprintf(config.ClusterNameTagKeyFormat, "awesome-cluster"): config.ClusterNameTagValue, + }, + }, + { + name: "without clusterName but ENV is set", + fields: fields{ + instanceID: "i-xxxxx", + nodeName: "fake-node", + clusterNameEnv: "awesome-cluster", + }, + want: map[string]string{ + config.ENIInstanceIDTag: "i-xxxxx", + config.ClusterNameTagKey: "awesome-cluster", }, }, { name: "with additional ENI tags", fields: fields{ instanceID: "i-xxxxx", + nodeName: "fake-node", additionalENITags: map[string]string{ "tagKey-1": "tagVal-1", "tagKey-2": "tagVal-2", }, }, want: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxxx", - "tagKey-1": "tagVal-1", - "tagKey-2": "tagVal-2", + config.ENIInstanceIDTag: "i-xxxxx", + "tagKey-1": "tagVal-1", + "tagKey-2": "tagVal-2", }, }, } @@ -1219,6 +1239,8 @@ func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { cache := &EC2InstanceMetadataCache{ instanceID: tt.fields.instanceID, clusterName: tt.fields.clusterName, + clusterNameEnvVal: tt.fields.clusterNameEnv, + nodeName: tt.fields.nodeName, additionalENITags: tt.fields.additionalENITags, } got := cache.buildENITags() @@ -1255,7 +1277,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1288,7 +1310,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1310,7 +1332,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1332,7 +1354,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1353,7 +1375,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1375,7 +1397,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1402,7 +1424,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1424,7 +1446,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1451,7 +1473,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1484,7 +1506,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1510,7 +1532,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1518,7 +1540,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Value: aws.String(tenMinuteAgo.Format(time.RFC3339)), }, { - Key: aws.String("cluster.k8s.amazonaws.com/name"), + Key: aws.String(config.ClusterNameTagKey), Value: aws.String("awesome-cluster"), }, }, @@ -1536,7 +1558,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1544,7 +1566,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Value: aws.String(tenMinuteAgo.Format(time.RFC3339)), }, { - Key: aws.String("cluster.k8s.amazonaws.com/name"), + Key: aws.String(config.ClusterNameTagKey), Value: aws.String("awesome-cluster"), }, }, @@ -1561,7 +1583,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1587,7 +1609,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1595,7 +1617,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Value: aws.String(tenMinuteAgo.Format(time.RFC3339)), }, { - Key: aws.String("cluster.k8s.amazonaws.com/name"), + Key: aws.String(config.ClusterNameTagKey), Value: aws.String("awesome-cluster"), }, }, @@ -1618,7 +1640,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: []*string{aws.String("node.k8s.amazonaws.com/instance_id")}, + Values: []*string{aws.String(config.ENIInstanceIDTag)}, }, { Name: aws.String("status"), @@ -1644,7 +1666,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Status: aws.String("available"), TagSet: []*ec2.Tag{ { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxxx"), }, { @@ -1652,7 +1674,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { Value: aws.String(now.Format(time.RFC3339)), }, { - Key: aws.String("cluster.k8s.amazonaws.com/name"), + Key: aws.String(config.ClusterNameTagKey), Value: aws.String("awesome-cluster"), }, }, @@ -1685,7 +1707,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { return nil }) } - cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, clusterName: tt.fields.clusterName, vpcID: vpcID} + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, clusterNameEnvVal: tt.fields.clusterName, vpcID: vpcID} got, err := cache.getLeakedENIs() if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) @@ -1705,6 +1727,8 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { type fields struct { instanceID string clusterName string + clusterNameEnvVal string + nodeName string additionalENITags map[string]string createTagsCalls []createTagsCall @@ -1724,19 +1748,29 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { fields: fields{ instanceID: "i-xxxx", clusterName: "awesome-cluster", + nodeName: "fake-node", createTagsCalls: []createTagsCall{ { input: &ec2.CreateTagsInput{ Resources: []*string{aws.String("eni-xxxx")}, Tags: []*ec2.Tag{ { - Key: aws.String("cluster.k8s.amazonaws.com/name"), - Value: aws.String("awesome-cluster"), + Key: aws.String(config.ENIOwnerTagKey), + Value: aws.String(config.ENIOwnerTagValue), }, { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(fmt.Sprintf(config.ClusterNameTagKeyFormat, "awesome-cluster")), + Value: aws.String(config.ClusterNameTagValue), + }, + { + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxx"), }, + + { + Key: aws.String(config.ENINodeNameTagKey), + Value: aws.String("fake-node"), + }, }, }, }, @@ -1753,13 +1787,16 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { fields: fields{ instanceID: "i-xxxx", clusterName: "awesome-cluster", + nodeName: "fake-node", createTagsCalls: nil, }, args: args{ eniID: "eni-xxxx", currentTags: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxx", - "cluster.k8s.amazonaws.com/name": "awesome-cluster", + config.ENIInstanceIDTag: "i-xxxx", + config.ENINodeNameTagKey: "fake-node", + fmt.Sprintf(config.ClusterNameTagKeyFormat, "awesome-cluster"): config.ClusterNameTagValue, + config.ENIOwnerTagKey: config.ENIOwnerTagValue, }, }, wantErr: nil, @@ -1769,13 +1806,77 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { fields: fields{ instanceID: "i-xxxx", clusterName: "awesome-cluster", + nodeName: "fake-node", + createTagsCalls: []createTagsCall{ + { + input: &ec2.CreateTagsInput{ + Resources: []*string{aws.String("eni-xxxx")}, + Tags: []*ec2.Tag{ + { + Key: aws.String(fmt.Sprintf(config.ClusterNameTagKeyFormat, "awesome-cluster")), + Value: aws.String(config.ClusterNameTagValue), + }, + { + Key: aws.String(config.ENINodeNameTagKey), + Value: aws.String("fake-node"), + }, + }, + }, + }, + }, + }, + args: args{ + eniID: "eni-xxxx", + currentTags: map[string]string{ + config.ENIInstanceIDTag: "i-xxxx", + config.ENIOwnerTagKey: config.ENIOwnerTagValue, + "anotherKey": "anotherDay", + }, + }, + wantErr: nil, + }, + { + name: "eni currently have partial tags, missing cluster name", + fields: fields{ + instanceID: "i-xxxx", + nodeName: "fake-node", + createTagsCalls: nil, + // []createTagsCall{ + // { + // input: &ec2.CreateTagsInput{ + // Resources: []*string{aws.String("eni-xxxx")}, + // Tags: []*ec2.Tag{ + // // { + // // Key: aws.String(config.ENINodeNameTagKey), + // // Value: aws.String("fake-node"), + // // }, + // }, + // }, + // }, + // }, + }, + args: args{ + eniID: "eni-xxxx", + currentTags: map[string]string{ + config.ENIInstanceIDTag: "i-xxxx", + "anotherKey": "anotherDay", + }, + }, + wantErr: nil, + }, + { + name: "eni currently have partial tags, missing cluster name from CNINode but ENV set", + fields: fields{ + instanceID: "i-xxxx", + nodeName: "fake-node", + clusterNameEnvVal: "awesome-cluster", createTagsCalls: []createTagsCall{ { input: &ec2.CreateTagsInput{ Resources: []*string{aws.String("eni-xxxx")}, Tags: []*ec2.Tag{ { - Key: aws.String("cluster.k8s.amazonaws.com/name"), + Key: aws.String(config.ClusterNameTagKey), Value: aws.String("awesome-cluster"), }, }, @@ -1786,8 +1887,8 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { args: args{ eniID: "eni-xxxx", currentTags: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxx", - "anotherKey": "anotherDay", + config.ENIInstanceIDTag: "i-xxxx", + "anotherKey": "anotherDay", }, }, wantErr: nil, @@ -1797,19 +1898,28 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { fields: fields{ instanceID: "i-xxxx", clusterName: "awesome-cluster", + nodeName: "fake-node", createTagsCalls: []createTagsCall{ { input: &ec2.CreateTagsInput{ Resources: []*string{aws.String("eni-xxxx")}, Tags: []*ec2.Tag{ { - Key: aws.String("cluster.k8s.amazonaws.com/name"), - Value: aws.String("awesome-cluster"), + Key: aws.String(config.ENIOwnerTagKey), + Value: aws.String(config.ENIOwnerTagValue), + }, + { + Key: aws.String(fmt.Sprintf(config.ClusterNameTagKeyFormat, "awesome-cluster")), + Value: aws.String(config.ClusterNameTagValue), }, { - Key: aws.String("node.k8s.amazonaws.com/instance_id"), + Key: aws.String(config.ENIInstanceIDTag), Value: aws.String("i-xxxx"), }, + { + Key: aws.String(config.ENINodeNameTagKey), + Value: aws.String("fake-node"), + }, }, }, err: errors.New("permission denied"), @@ -1836,6 +1946,8 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { ec2SVC: mockEC2, instanceID: tt.fields.instanceID, clusterName: tt.fields.clusterName, + clusterNameEnvVal: tt.fields.clusterNameEnvVal, + nodeName: tt.fields.nodeName, additionalENITags: tt.fields.additionalENITags, } err := cache.TagENI(tt.args.eniID, tt.args.currentTags) diff --git a/pkg/config/type.go b/pkg/config/type.go new file mode 100644 index 0000000000..ec658b2caf --- /dev/null +++ b/pkg/config/type.go @@ -0,0 +1,32 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +// Constant values used in aws-node +// TODO: consolidate all constants in the project +const ( + // Cluster name ENV + ClusterNameEnv = "CLUSTER_NAME" + // ENI tags + ClusterNameTagKeyFormat = "kubernetes.io/cluster/%s" + ClusterNameTagValue = "owned" + + ClusterNameTagKey = "cluster.k8s.amazonaws.com/name" + ENIInstanceIDTag = "node.k8s.amazonaws.com/instance_id" + ENINodeNameTagKey = "node.k8s.amazonaws.com/nodename" + + // ENI owner tag + ENIOwnerTagKey = "eks:eni:owner" + ENIOwnerTagValue = "amazon-vpc-cni" +) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 33f75900f5..cb11da3811 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/util/retry" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" + "github.com/aws/amazon-vpc-cni-k8s/pkg/config" "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" @@ -166,8 +167,6 @@ const ( // envManageUntaggedENI is used to determine if untagged ENIs should be managed or unmanaged envManageUntaggedENI = "MANAGE_UNTAGGED_ENI" - eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" - // envAnnotatePodIP is used to annotate[vpc.amazonaws.com/pod-ips] pod's with IPs // Ref : https://github.com/projectcalico/calico/issues/3530 // not present; in which case we fall back to the k8s podIP @@ -246,7 +245,7 @@ func (c *IPAMContext) setUnmanagedENIs(tagMap map[string]awsutils.TagMap) { if tags[eniNoManageTagKey] != "true" { continue } - } else if _, found := tags[eniNodeTagKey]; found && tags[eniNodeTagKey] == c.awsClient.GetInstanceID() { + } else if _, found := tags[config.ENIInstanceIDTag]; found && tags[config.ENIInstanceIDTag] == c.awsClient.GetInstanceID() { continue } else if c.enableManageUntaggedMode { continue @@ -342,7 +341,16 @@ func New(k8sClient client.Client) (*IPAMContext, error) { c.enableIPv4 = isIPv4Enabled() c.enableIPv6 = isIPv6Enabled() c.disableENIProvisioning = disableENIProvisioning() - client, err := awsutils.New(c.useSubnetDiscovery, c.useCustomNetworking, disableLeakedENICleanup(), c.enableIPv4, c.enableIPv6) + c.myNodeName = os.Getenv(envNodeName) + + var clusterName string + clusterName, err := getClusterName(c.k8sClient, c.myNodeName) + if err != nil { + // only log the error, fallback to running cleanup routine on the aws-node + log.Error("failed to get cluster name from CNINode") + } + + client, err := awsutils.New(c.useSubnetDiscovery, c.useCustomNetworking, disableLeakedENICleanup(clusterName), c.enableIPv4, c.enableIPv6, clusterName, c.myNodeName) if err != nil { return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } @@ -377,7 +385,7 @@ func New(k8sClient client.Client) (*IPAMContext, error) { } c.awsClient.InitCachedPrefixDelegation(c.enablePrefixDelegation) - c.myNodeName = os.Getenv(envNodeName) + checkpointer := datastore.NewJSONFile(dsBackingStorePath()) c.dataStore = datastore.NewDataStore(log, checkpointer, c.enablePrefixDelegation) @@ -1743,7 +1751,13 @@ func disableENIProvisioning() bool { return utils.GetBoolAsStringEnvVar(envDisableENIProvisioning, false) } -func disableLeakedENICleanup() bool { +func disableLeakedENICleanup(clusterName string) bool { + + // cluster name is read from the CNINode CRD created by vpc-resource-controller and if found controller will run the cleanup routine to delete leaked ENIs + // so set disable leaked ENI cleanup to true on aws-node + if clusterName != "" { + return true + } // Cases where leaked ENI cleanup is disabled: // 1. IPv6 is enabled, so no ENIs are attached // 2. ENI provisioning is disabled, so ENIs are not managed by IPAMD @@ -2335,3 +2349,21 @@ func (c *IPAMContext) AddFeatureToCNINode(ctx context.Context, featureName rcv1a newCNINode.Spec.Features = append(newCNINode.Spec.Features, newFeature) return c.k8sClient.Patch(ctx, newCNINode, client.MergeFromWithOptions(cniNode, client.MergeFromWithOptimisticLock{})) } + +// getClusterName returns the cluster name by reading CNINode Tags field +func getClusterName(k8sClient client.Client, nodeName string) (string, error) { + cniNode := &rcv1alpha1.CNINode{} + err := retry.OnError(retry.DefaultBackoff, func(error) bool { return true }, + func() error { + return k8sClient.Get(context.TODO(), types.NamespacedName{Name: nodeName}, cniNode) + }) + if err != nil { + return "", errors.Wrap(err, "failed to get CNINode") + } + + if val, ok := cniNode.Spec.Tags[config.ClusterNameTagKey]; ok { + return val, nil + } + + return "", fmt.Errorf("cluster name tag not found in CNINode") +} diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 7dc6133a34..277e06d6d3 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -30,7 +30,6 @@ import ( "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,10 +39,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks" + "github.com/aws/amazon-vpc-cni-k8s/pkg/config" mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks" @@ -80,6 +79,7 @@ const ( v6prefix01 = "2001:db8::/64" instanceID = "i-0e1f3b9eb950e4980" externalEniConfigLabel = "vpc.amazonaws.com/externalEniConfig" + clusterName = "fake-cluster" ) type testMocks struct { @@ -520,7 +520,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool) testAddr12 := ipaddr12 eni2 := secENIid - podENIConfig := &v1alpha1.ENIConfigSpec{ + podENIConfig := &eniconfigscheme.ENIConfigSpec{ SecurityGroups: []string{"sg1-id", "sg2-id"}, Subnet: "subnet1", } @@ -587,15 +587,15 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool) Status: v1.NodeStatus{}, } if unschedulabeNode { - fakeNode.Spec.Taints = append(fakeNode.Spec.Taints, corev1.Taint{ + fakeNode.Spec.Taints = append(fakeNode.Spec.Taints, v1.Taint{ Key: "node.kubernetes.io/unschedulable", - Effect: corev1.TaintEffectNoSchedule, + Effect: v1.TaintEffectNoSchedule, }) } m.k8sClient.Create(ctx, &fakeNode) // Create a dummy ENIConfig - fakeENIConfig := v1alpha1.ENIConfig{ + fakeENIConfig := eniconfigscheme.ENIConfig{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "az1"}, Spec: eniconfigscheme.ENIConfigSpec{ @@ -665,7 +665,7 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig bool) { testPrefix2 := prefix02 eni2 := secENIid - podENIConfig := &v1alpha1.ENIConfigSpec{ + podENIConfig := &eniconfigscheme.ENIConfigSpec{ SecurityGroups: []string{"sg1-id", "sg2-id"}, Subnet: "subnet1", } @@ -736,7 +736,7 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig bool) { m.k8sClient.Create(ctx, &fakeNode) //Create a dummy ENIConfig - fakeENIConfig := v1alpha1.ENIConfig{ + fakeENIConfig := eniconfigscheme.ENIConfig{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "az1"}, Spec: eniconfigscheme.ENIConfigSpec{ @@ -1410,10 +1410,10 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { eni3.ENIID: {"hi": "tag", eniNoManageTagKey: "false"}} Test4TagMap := map[string]awsutils.TagMap{ eni2.ENIID: {"hi": "tag", eniNoManageTagKey: "true"}, - eni3.ENIID: {"hi": "tag", eniNodeTagKey: instanceID}} + eni3.ENIID: {"hi": "tag", config.ENIInstanceIDTag: instanceID}} Test5TagMap := map[string]awsutils.TagMap{ - eni2.ENIID: {"hi": "tag", eniNodeTagKey: "i-abcdabcdabcd"}, - eni3.ENIID: {"hi": "tag", eniNodeTagKey: instanceID}} + eni2.ENIID: {"hi": "tag", config.ENIInstanceIDTag: "i-abcdabcdabcd"}, + eni3.ENIID: {"hi": "tag", config.ENIInstanceIDTag: instanceID}} tests := []struct { name string @@ -1462,7 +1462,7 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { if tags[eniNoManageTagKey] == "true" { return true } - } else if _, ok := tags[eniNodeTagKey]; ok && tags[eniNodeTagKey] != instanceID { + } else if _, ok := tags[config.ENIInstanceIDTag]; ok && tags[config.ENIInstanceIDTag] != instanceID { return true } } @@ -1498,10 +1498,10 @@ func TestIPAMContext_filterUnmanagedENIs_disableManageUntaggedMode(t *testing.T) eni3.ENIID: {"hi": "tag", eniNoManageTagKey: "false"}} Test4TagMap := map[string]awsutils.TagMap{ eni2.ENIID: {"hi": "tag", eniNoManageTagKey: "true"}, - eni3.ENIID: {"hi": "tag", eniNodeTagKey: instanceID}} + eni3.ENIID: {"hi": "tag", config.ENIInstanceIDTag: instanceID}} Test5TagMap := map[string]awsutils.TagMap{ - eni2.ENIID: {"hi": "tag", eniNodeTagKey: "i-abcdabcdabcd"}, - eni3.ENIID: {"hi": "tag", eniNodeTagKey: instanceID}} + eni2.ENIID: {"hi": "tag", config.ENIInstanceIDTag: "i-abcdabcdabcd"}, + eni3.ENIID: {"hi": "tag", config.ENIInstanceIDTag: instanceID}} tests := []struct { name string @@ -1553,7 +1553,7 @@ func TestIPAMContext_filterUnmanagedENIs_disableManageUntaggedMode(t *testing.T) if tags[eniNoManageTagKey] == "true" { return true } - } else if _, ok := tags[eniNodeTagKey]; ok && tags[eniNodeTagKey] != instanceID { + } else if _, ok := tags[config.ENIInstanceIDTag]; ok && tags[config.ENIInstanceIDTag] != instanceID { return true } } @@ -1998,7 +1998,7 @@ func TestIPAMContext_enableSecurityGroupsForPods(t *testing.T) { mockContext.enablePodENI = true mockContext.tryEnableSecurityGroupsForPods(ctx) - var notUpdatedNode corev1.Node + var notUpdatedNode v1.Node NodeKey := types.NamespacedName{ Namespace: "", Name: myNodeName, @@ -2174,7 +2174,7 @@ func TestAnnotatePod(t *testing.T) { ctx := context.Background() // Define the Pod objects to test - pod := corev1.Pod{ + pod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: "test-namespace", @@ -2362,3 +2362,75 @@ func TestAddFeatureToCNINode(t *testing.T) { }) } } + +func TestGetClusterName(t *testing.T) { + type args struct { + nodeName string + cniNode *rcscheme.CNINode + } + tests := []struct { + name string + expectedClusterName string + args args + wantErr bool + }{ + { + name: "CNINode contains cluster name tag", + expectedClusterName: clusterName, + args: args{ + nodeName: myNodeName, + cniNode: &rcscheme.CNINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: myNodeName, + Namespace: "", + }, + Spec: rcscheme.CNINodeSpec{ + Tags: map[string]string{ + config.ClusterNameTagKey: clusterName, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "CNINode does not contain cluster name", + expectedClusterName: "", + args: args{ + nodeName: myNodeName, + cniNode: &rcscheme.CNINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: myNodeName, + Namespace: "", + }, + }, + }, + wantErr: true, + }, + { + name: "CNINode does not exist", + expectedClusterName: "", + args: args{ + nodeName: myNodeName, + cniNode: &rcscheme.CNINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-node", + Namespace: "", + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + m.k8sClient.Create(context.Background(), tt.args.cniNode) + clusterName, err := getClusterName(m.k8sClient, tt.args.nodeName) + assert.Equal(t, tt.expectedClusterName, clusterName) + assert.Equal(t, err != nil, tt.wantErr) + }) + } +} diff --git a/test/framework/resources/k8s/manager.go b/test/framework/resources/k8s/manager.go index 23de806201..13e7003c61 100644 --- a/test/framework/resources/k8s/manager.go +++ b/test/framework/resources/k8s/manager.go @@ -34,6 +34,7 @@ type ResourceManagers interface { ConfigMapManager() resources.ConfigMapManager NetworkPolicyManager() resources.NetworkPolicyManager EventManager() resources.EventManager + CNINodeManager() resources.CNINodeManager } type defaultManager struct { @@ -48,6 +49,7 @@ type defaultManager struct { configMapManager resources.ConfigMapManager networkPolicyManager resources.NetworkPolicyManager eventManager resources.EventManager + cniNodeManager resources.CNINodeManager } func NewResourceManager(k8sClient client.Client, k8sClientset *kubernetes.Clientset, scheme *runtime.Scheme, config *rest.Config) ResourceManagers { @@ -63,6 +65,7 @@ func NewResourceManager(k8sClient client.Client, k8sClientset *kubernetes.Client configMapManager: resources.NewConfigMapManager(k8sClient), networkPolicyManager: resources.NewNetworkPolicyManager(k8sClient), eventManager: resources.NewEventManager(k8sClient), + cniNodeManager: resources.NewCNINodeManager(k8sClient), } } @@ -109,3 +112,7 @@ func (m *defaultManager) NetworkPolicyManager() resources.NetworkPolicyManager { func (m defaultManager) EventManager() resources.EventManager { return m.eventManager } + +func (m *defaultManager) CNINodeManager() resources.CNINodeManager { + return m.cniNodeManager +} diff --git a/test/framework/resources/k8s/resources/cninode.go b/test/framework/resources/k8s/resources/cninode.go new file mode 100644 index 0000000000..c74141a34d --- /dev/null +++ b/test/framework/resources/k8s/resources/cninode.go @@ -0,0 +1,41 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resources + +import ( + "context" + + rcv1alpha1 "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type CNINodeManager interface { + GetCNINode(nodeName string) (*rcv1alpha1.CNINode, error) +} + +type defaultCNINodeManager struct { + k8sClient client.Client +} + +func (c defaultCNINodeManager) GetCNINode(nodeName string) (*rcv1alpha1.CNINode, error) { + cniNode := &rcv1alpha1.CNINode{} + err := c.k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeName}, cniNode) + return cniNode, err + +} + +func NewCNINodeManager(k8sClient client.Client) CNINodeManager { + return &defaultCNINodeManager{k8sClient: k8sClient} +} diff --git a/test/integration/ipamd/cninode_test.go b/test/integration/ipamd/cninode_test.go new file mode 100644 index 0000000000..f65ba683a9 --- /dev/null +++ b/test/integration/ipamd/cninode_test.go @@ -0,0 +1,36 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "github.com/aws/amazon-vpc-cni-k8s/pkg/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CNINode Validation", func() { + Describe("Validate CNINode contains cluster name tag", func() { + Context("when nodes are ready", func() { + It("should have the cluster name tag populated", func() { + By("getting CNINode for the primary node and verify cluster name tag exists") + cniNode, err := f.K8sResourceManagers.CNINodeManager().GetCNINode(primaryNode.Name) + Expect(err).ToNot(HaveOccurred()) + val, ok := cniNode.Spec.Tags[config.ClusterNameTagKey] + Expect(ok).To(BeTrue()) + Expect(val).To(Equal(f.Options.ClusterName)) + }) + }) + }) +}) diff --git a/test/integration/ipamd/eni_ip_leak_test.go b/test/integration/ipamd/eni_ip_leak_test.go index 0e765c6425..8257bac7d5 100644 --- a/test/integration/ipamd/eni_ip_leak_test.go +++ b/test/integration/ipamd/eni_ip_leak_test.go @@ -3,8 +3,6 @@ package ipamd import ( "time" - v1 "k8s.io/api/core/v1" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -13,7 +11,6 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" ) -var primaryNode v1.Node var numOfNodes int var _ = Describe("[CANARY][SMOKE] ENI/IP Leak Test", func() { diff --git a/test/integration/ipamd/eni_tag_test.go b/test/integration/ipamd/eni_tag_test.go index 661f900c5e..07924bf088 100644 --- a/test/integration/ipamd/eni_tag_test.go +++ b/test/integration/ipamd/eni_tag_test.go @@ -132,6 +132,24 @@ var _ = Describe("test tags are created on Secondary ENI", func() { VerifyTagIsPresentOnENIs(newENIs, expectedTags) }) }) + Context("when additional secondary ENI are created without setting CLUSTER_NAME", func() { + BeforeEach(func() { + expectedTags = map[string]string{ + "kubernetes.io/cluster/" + f.Options.ClusterName: "owned", + "node.k8s.amazonaws.com/nodename": primaryNode.Name, + "eks:eni:owner": "amazon-vpc-cni", + } + + environmentVariables = map[string]string{ + "WARM_ENI_TARGET": "2", + } + }) + + It("new secondary ENI should have cluster name tags read from CNINode", func() { + Skip("skip till vpc-resource-controller release") + VerifyTagIsPresentOnENIs(newENIs, expectedTags) + }) + }) }) // VerifyTagIsPresentOnENIs verifies that the list of ENIs have expected tag key-val pair diff --git a/test/integration/ipamd/ipamd_suite_test.go b/test/integration/ipamd/ipamd_suite_test.go index 2caca00f83..44ed2bb242 100644 --- a/test/integration/ipamd/ipamd_suite_test.go +++ b/test/integration/ipamd/ipamd_suite_test.go @@ -34,6 +34,7 @@ const ( ) var coreDNSDeploymentCopy *v1.Deployment +var primaryNode *corev1.Node func TestIPAMD(t *testing.T) { RegisterFailHandler(Fail) @@ -56,7 +57,6 @@ var _ = BeforeSuite(func() { // Nominate the first untainted node as the one to run coredns deployment against By("adding nodeSelector in coredns deployment to be scheduled on single node") - var primaryNode *corev1.Node for _, n := range nodeList.Items { if len(n.Spec.Taints) == 0 { primaryNode = &n