diff --git a/cmd/aws-vpc-cni/main.go b/cmd/aws-vpc-cni/main.go index 9455c90af1..c76d93a83f 100644 --- a/cmd/aws-vpc-cni/main.go +++ b/cmd/aws-vpc-cni/main.go @@ -36,7 +36,6 @@ package main import ( - "bytes" "encoding/json" "net" "os" @@ -171,13 +170,11 @@ type Range struct { Gateway net.IP `json:"gateway,omitempty"` } +// Wait for IPAMD health check to pass. Note that if IPAMD fails to start, wait happens indefinitely until liveness probe kills pod func waitForIPAM() bool { for { cmd := exec.Command("./grpc-health-probe", "-addr", "127.0.0.1:50051", ">", "/dev/null", "2>&1") - var outb bytes.Buffer - cmd.Stdout = &outb - cmd.Run() - if outb.String() == "" { + if err := cmd.Run(); err == nil { return true } } @@ -443,7 +440,7 @@ func _main() int { err = cp.CopyFile(tmpAWSconflistFile, defaultHostCNIConfDirPath+awsConflistFile) if err != nil { - log.WithError(err).Errorf("Failed to copy 10-awsconflist") + log.WithError(err).Errorf("Failed to copy %s", awsConflistFile) return 1 } log.Infof("Successfully copied CNI plugin binary and config file.") diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 3e3e54f908..ab765e3cb4 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -397,7 +397,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) // New creates an EC2InstanceMetadataCache func New(useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool, eventRecorder *eventrecorder.EventRecorder) (*EC2InstanceMetadataCache, error) { - //ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run + // ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run ctx := context.Background() // Initializes prometheus metrics diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 329f78be90..f3b1a4a6d7 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -184,13 +184,13 @@ type AddressInfo struct { // CidrInfo type CidrInfo struct { - //Either v4/v6 Host or LPM Prefix + // Either v4/v6 Host or LPM Prefix Cidr net.IPNet - //Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6) + // Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6) IPAddresses map[string]*AddressInfo - //true if Cidr here is an LPM prefix + // true if Cidr here is an LPM prefix IsPrefix bool - //IP Address Family of the Cidr + // IP Address Family of the Cidr AddressFamily string } @@ -200,8 +200,8 @@ func (cidr *CidrInfo) Size() int { } func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { - //Either v4 or v6 for now. - //Check in V4 prefixes + // Either v4 or v6 for now. + // Check in V4 prefixes for _, availableCidr := range e.AvailableIPv4Cidrs { for _, addr := range availableCidr.IPAddresses { if addr.IPAMKey == ipamKey { @@ -210,7 +210,7 @@ func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { } } - //Check in V6 prefixes + // Check in V6 prefixes for _, availableCidr := range e.IPv6Cidrs { for _, addr := range availableCidr.IPAddresses { if addr.IPAMKey == ipamKey { @@ -495,7 +495,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("DataStore Add an ENI %s", eniID) + ds.log.Debugf("DataStore add an ENI %s", eniID) _, ok := ds.eniPool[eniID] if ok { @@ -712,7 +712,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned) + ds.log.Debugf("AssignIPv4Address: IP address pool stats: total %d, assigned %d", ds.total, ds.assigned) if eni, _, addr := ds.eniPool.FindAddressForSandbox(ipamKey); addr != nil { ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for sandbox %s", ipamKey) @@ -1123,8 +1123,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total:%d, assigned %d, sandbox %s", - ds.total, ds.assigned, ipamKey) + ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total %d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey) eni, availableCidr, addr := ds.eniPool.FindAddressForSandbox(ipamKey) if addr == nil { @@ -1241,7 +1240,7 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { tmpENIInfo := *eniInfo tmpENIInfo.AvailableIPv4Cidrs = make(map[string]*CidrInfo, len(eniInfo.AvailableIPv4Cidrs)) tmpENIInfo.IPv6Cidrs = make(map[string]*CidrInfo, len(eniInfo.IPv6Cidrs)) - for cidr, _ := range eniInfo.AvailableIPv4Cidrs { + for cidr := range eniInfo.AvailableIPv4Cidrs { tmpENIInfo.AvailableIPv4Cidrs[cidr] = &CidrInfo{ Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr, IPAddresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)), @@ -1253,7 +1252,7 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { tmpENIInfo.AvailableIPv4Cidrs[cidr].IPAddresses[ip] = &ipAddrInfo } } - for cidr, _ := range eniInfo.IPv6Cidrs { + for cidr := range eniInfo.IPv6Cidrs { tmpENIInfo.IPv6Cidrs[cidr] = &CidrInfo{ Cidr: eniInfo.IPv6Cidrs[cidr].Cidr, IPAddresses: make(map[string]*AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)), diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index a6687b86ac..01753d17e3 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -412,7 +412,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.primaryIP = make(map[string]string) c.reconcileCooldownCache.cache = make(map[string]time.Time) - //WARM and Min IP/Prefix targets are ignored in IPv6 mode + // WARM and Min IP/Prefix targets are ignored in IPv6 mode c.warmENITarget = getWarmENITarget() c.warmIPTarget = getWarmIPTarget() c.minimumIPTarget = getMinimumIPTarget() @@ -428,8 +428,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex return nil, err } - //Let's validate if the configured combination of env variables is supported before we - //proceed any further + // Validate if the configured combination of env variables is supported before proceeding further if !c.isConfigValid() { return nil, fmt.Errorf("ipamd: failed to validate configuration") } @@ -446,7 +445,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex mac := c.awsClient.GetPrimaryENImac() - // retrieve security groups + // Retrieve security groups if c.enableIPv4 && !c.disableENIProvisioning { err = c.awsClient.RefreshSGIDs(mac) if err != nil { @@ -476,16 +475,15 @@ func (c *IPAMContext) nodeInit() error { } if c.enableIPv4 { - //Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer - //be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled. + // Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer + // be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled. vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs() if err != nil { return err } } - err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4, - c.enableIPv6) + err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4, c.enableIPv6) if err != nil { return errors.Wrap(err, "ipamd init: failed to set up host network") } @@ -502,7 +500,6 @@ func (c *IPAMContext) nodeInit() error { for _, eni := range enis { log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID) - isTrunkENI := eni.ENIID == metadataResult.TrunkENI isEFAENI := metadataResult.EFAENIs[eni.ENIID] if !isTrunkENI && !c.disableENIProvisioning { @@ -542,22 +539,22 @@ func (c *IPAMContext) nodeInit() error { } if c.enableIPv6 { - //We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always - //start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation - //feature in IPv6 mode as we don't support (yet) a non-PD v6 option. In addition, we don't support custom - //networking & SGPP in IPv6 mode yet. So, we will skip the corresponding setup. Will save us from checking - //if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away - //with this check and not change anything else in the below setup. + // We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always + // start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation + // feature in IPv6 mode as we don't support (yet) a non-PD v6 option. In addition, we don't support custom + // networking & SGPP in IPv6 mode yet. So, we will skip the corresponding setup. Will save us from checking + // if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away + // with this check and not change anything else in the below setup. return nil } if c.enablePrefixDelegation { - //During upgrade or if prefix delgation knob is disabled to enabled then we - //might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on + // During upgrade or if prefix delgation knob is disabled to enabled then we + // might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on c.tryUnassignIPsFromENIs() } else { - //When prefix delegation knob is enabled to disabled then we might - //have unused prefixes attached to the ENIs so need to cleanup + // When prefix delegation knob is enabled to disabled then we might + // have unused prefixes attached to the ENIs so need to cleanup c.tryUnassignPrefixesFromENIs() } @@ -602,21 +599,22 @@ func (c *IPAMContext) nodeInit() error { c.askForTrunkENIIfNeeded(ctx) } - if !c.disableENIProvisioning { - // For a new node, attach Cidrs (secondary ips/prefixes) - increasedPool, err := c.tryAssignCidrs() - if err == nil && increasedPool { - c.updateLastNodeIPPoolAction() - } else if err != nil { - if containsInsufficientCIDRsOrSubnetIPs(err) { - log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") - c.lastInsufficientCidrError = time.Now() - return nil - } - return err + // On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs. + if !c.disableENIProvisioning && c.isDatastorePoolTooLow() { + if err := c.increaseDatastorePool(ctx); err != nil { + // Note that the only error currently returned by increaseDatastorePool is an error attaching CIDRs (other than insufficient IPs) + podENIErrInc("nodeInit") + return errors.New("error while trying to increase datastore pool") + } + // If custom networking is enabled and the pool is empty, return an error, as there is a misconfiguration and + // the node should not become ready. + if c.useCustomNetworking && c.isDatastorePoolEmpty() { + podENIErrInc("nodeInit") + return errors.New("Failed to attach any ENIs for custom networking") } } + log.Debug("node init completed successfully") return nil } @@ -805,7 +803,7 @@ func (c *IPAMContext) tryUnassignCidrsFromAll() { } } -func (c *IPAMContext) increaseDatastorePool(ctx context.Context) { +func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error { log.Debug("Starting to increase pool size") ipamdActionsInprogress.WithLabelValues("increaseDatastorePool").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("increaseDatastorePool").Sub(float64(1)) @@ -813,40 +811,41 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) { short, _, warmIPTargetDefined := c.datastoreTargetState() if warmIPTargetDefined && short == 0 { log.Debugf("Skipping increase Datastore pool, warm target reached") - return + return nil } if !warmIPTargetDefined { shortPrefix, warmTargetDefined := c.datastorePrefixTargetState() if warmTargetDefined && shortPrefix == 0 { log.Debugf("Skipping increase Datastore pool, warm prefix target reached") - return + return nil } } if c.isTerminating() { log.Debug("AWS CNI is terminating, will not try to attach any new IPs or ENIs right now") - return + return nil } if !c.manageENIsNonScheduleable && c.isNodeNonSchedulable() { log.Debug("AWS CNI is on a non schedulable node, will not try to attach any new IPs or ENIs right now") - return + return nil } // Try to add more Cidrs to existing ENIs first. if c.inInsufficientCidrCoolingPeriod() { log.Debugf("Recently we had InsufficientCidr error hence will wait for %v before retrying", insufficientCidrErrorCooldown) - return + return nil } increasedPool, err := c.tryAssignCidrs() if err != nil { - log.Errorf(err.Error()) if containsInsufficientCIDRsOrSubnetIPs(err) { log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") c.lastInsufficientCidrError = time.Now() - return + return nil } + log.Errorf(err.Error()) + return err } if increasedPool { c.updateLastNodeIPPoolAction() @@ -856,16 +855,20 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) { if c.enablePodENI && c.dataStore.GetTrunkENI() == "" { reserveSlotForTrunkENI = 1 } - // If we did not add an IP, try to add an ENI instead. + // If we did not add any IPs, try to allocate an ENI. if c.dataStore.GetENIs() < (c.maxENI - c.unmanagedENI - reserveSlotForTrunkENI) { if err = c.tryAllocateENI(ctx); err == nil { c.updateLastNodeIPPoolAction() + } else { + // Note that no error is returned if ENI allocation fails. This is because ENI allocation failure should not cause node to be "NotReady". + log.Debugf("Error trying to allocate ENI: %v", err) } } else { log.Debugf("Skipping ENI allocation as the max ENI limit of %d is already reached (accounting for %d unmanaged ENIs and %d trunk ENIs)", c.maxENI, c.unmanagedENI, reserveSlotForTrunkENI) } } + return nil } func (c *IPAMContext) updateLastNodeIPPoolAction() { @@ -1113,9 +1116,9 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address() if c.enableIPv6 && eni == primaryENI { - //In v6 PD Mode, VPC CNI will only manage primary ENI. Once we start supporting secondary IP and custom - //networking modes for v6, we will relax this restriction. We filter out all the ENIs except Primary ENI - //in v6 mode (prior to landing here), but included the primary ENI check as a safety net. + // In v6 PD Mode, VPC CNI will only manage primary ENI. Once we start supporting secondary IP and custom + // networking modes for v6, we will relax this restriction. We filter out all the ENIs except Primary ENI + // in v6 mode (prior to landing here), but included the primary ENI check as a safety net. err := c.assignIPv6Prefix(eni) if err != nil { return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to Primary ENI") @@ -1135,7 +1138,7 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT } } log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes)) - //Either case add the IPs and prefixes to datastore. + // Either case add the IPs and prefixes to datastore. c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni) c.addENIv4prefixesToDataStore(eniMetadata.IPv4Prefixes, eni) } @@ -1144,7 +1147,7 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT } func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []*ec2.NetworkInterfacePrivateIpAddress, eni string) { - //Add all the secondary IPs + // Add all the secondary IPs for _, ec2PrivateIpAddr := range ec2PrivateIpAddrs { if aws.BoolValue(ec2PrivateIpAddr.Primary) { continue @@ -1161,8 +1164,7 @@ func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []*ec2.Net } func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4PrefixSpecification, eni string) { - - //Walk thru all prefixes + // Walk thru all prefixes for _, ec2PrefixAddr := range ec2PrefixAddrs { strIpv4Prefix := aws.StringValue(ec2PrefixAddr.Ipv4Prefix) _, ipnet, err := net.ParseCIDR(strIpv4Prefix) @@ -1184,12 +1186,12 @@ func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4Pref func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv6PrefixSpecification, eni string) { log.Debugf("Updating datastore with IPv6Prefix(es) for ENI: %v, count: %v", eni, len(ec2PrefixAddrs)) - //Walk through all prefixes + // Walk through all prefixes for _, ec2PrefixAddr := range ec2PrefixAddrs { strIpv6Prefix := aws.StringValue(ec2PrefixAddr.Ipv6Prefix) _, ipnet, err := net.ParseCIDR(strIpv6Prefix) if err != nil { - //Parsing failed, get next prefix + // Parsing failed, get next prefix log.Debugf("Parsing failed, moving on to next prefix") continue } @@ -1353,8 +1355,11 @@ func podENIErrInc(fn string) { // nodeIPPoolReconcile reconcile ENI and IP info from metadata service and IP addresses in datastore func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Duration) { + // To reduce the number of EC2 API calls, skip reconciliation if IPs were recently added to the datastore. timeSinceLast := time.Since(c.lastNodeIPPoolAction) - if timeSinceLast <= interval { + // Make an exception if node needs a trunk ENI and one is not currently attached. + needsTrunkEni := c.enablePodENI && c.dataStore.GetTrunkENI() == "" + if timeSinceLast <= interval && !needsTrunkEni { return } @@ -1877,7 +1882,6 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) overPrefix = max(min(overPrefix, stats.TotalPrefixes-prefixNeededForMinIP), 0) log.Debugf("Current warm IP stats : target: %d, short(prefixes): %d, over(prefixes): %d, stats: %s", c.warmIPTarget, shortPrefix, overPrefix, stats) return shortPrefix, overPrefix, true - } log.Debugf("Current warm IP stats : target: %d, short: %d, over: %d, stats: %s", c.warmIPTarget, short, over, stats) @@ -2198,6 +2202,11 @@ func (c *IPAMContext) GetIPv4Limit() (int, int, error) { return maxIPsPerENI, maxPrefixesPerENI, nil } +func (c *IPAMContext) isDatastorePoolEmpty() bool { + stats := c.dataStore.GetIPStats(ipV4AddrFamily) + return stats.TotalIPs == 0 +} + func (c *IPAMContext) isDatastorePoolTooLow() bool { short, _, warmTargetDefined := c.datastoreTargetState() if warmTargetDefined { @@ -2222,7 +2231,6 @@ func (c *IPAMContext) isDatastorePoolTooLow() bool { c.logPoolStats(stats) } return poolTooLow - } func (c *IPAMContext) isDatastorePoolTooHigh() bool { @@ -2316,10 +2324,9 @@ func (c *IPAMContext) initENIAndIPLimits() (err error) { log.Debugf("Max ip per ENI %d and max prefixes per ENI %d", c.maxIPsPerENI, c.maxPrefixesPerENI) } - //WARM and MAX ENI & IP/Prefix counts are no-op in IPv6 Prefix delegation mode. Once we start supporting IPv6 in - //Secondary IP mode, these variables will play the same role as they currently do in IPv4 mode. So, for now we - //leave them at their default values. - + // WARM and MAX ENI & IP/Prefix counts are no-op in IPv6 Prefix delegation mode. Once we start supporting IPv6 in + // Secondary IP mode, these variables will play the same role as they currently do in IPv4 mode. So, for now we + // leave them at their default values. return nil } diff --git a/scripts/run-ginkgo-integration-suite.sh b/scripts/run-ginkgo-integration-suite.sh index 041e72296e..b65bdb64cd 100755 --- a/scripts/run-ginkgo-integration-suite.sh +++ b/scripts/run-ginkgo-integration-suite.sh @@ -36,7 +36,7 @@ function load_test_parameters(){ } function run_ginkgo_test() { - (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v --timeout 30m --no-color --fail-on-pending $GINKGO_TEST_BUILD/$SUITE_NAME.test -- \ + (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v --timeout 60m --no-color --fail-on-pending $GINKGO_TEST_BUILD/$SUITE_NAME.test -- \ --cluster-kubeconfig="$KUBE_CONFIG_PATH" \ --cluster-name="$CLUSTER_NAME" \ --aws-region="$REGION" \ diff --git a/test/framework/options.go b/test/framework/options.go index 0449a8bec2..d0cbfdc424 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -46,6 +46,7 @@ type Options struct { PrivateSubnets string AvailabilityZones string PublicRouteTableID string + NgK8SVersion string } func (options *Options) BindFlags() { @@ -68,6 +69,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.PrivateSubnets, "private-subnets", "", "Comma separated list of private subnets (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)") flag.StringVar(&options.AvailabilityZones, "availability-zones", "", "Comma separated list of private subnets (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)") flag.StringVar(&options.PublicRouteTableID, "public-route-table-id", "", "Public route table ID (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)") + flag.StringVar(&options.NgK8SVersion, "ng-kubernetes-version", "1.25", `Kubernetes version for self-managed node groups (optional, default is "1.25")`) } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go index 26330f65c6..9847621e9a 100644 --- a/test/framework/resources/aws/utils/nodegroup.go +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -29,11 +29,11 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" ) -const CreateNodeGroupCFNTemplate = "/testdata/amazon-eks-nodegroup.yaml" - -// Docker will be default, if not specified const ( - CONTAINERD = "containerd" + // Docker will be default, if not specified + CONTAINERD = "containerd" + CreateNodeGroupCFNTemplate = "/testdata/amazon-eks-nodegroup.yaml" + NodeImageIdSSMParam = "/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id" ) type NodeGroupProperties struct { @@ -124,6 +124,10 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node ParameterKey: aws.String("NodeGroupName"), ParameterValue: aws.String(properties.NodeGroupName), }, + { + ParameterKey: aws.String("NodeImageIdSSMParam"), + ParameterValue: aws.String(fmt.Sprintf(NodeImageIdSSMParam, f.Options.NgK8SVersion)), + }, { ParameterKey: aws.String("NodeAutoScalingGroupMinSize"), ParameterValue: aws.String(asgSizeString), diff --git a/test/framework/resources/k8s/resources/node.go b/test/framework/resources/k8s/resources/node.go index 5c5d3ac8f4..a704509cf9 100644 --- a/test/framework/resources/k8s/resources/node.go +++ b/test/framework/resources/k8s/resources/node.go @@ -89,8 +89,6 @@ func (d *defaultNodeManager) WaitTillNodesReady(nodeLabelKey string, nodeLabelVa } } } - return true, nil - }, context.Background().Done()) } diff --git a/test/integration/custom-networking/custom_networking_test.go b/test/integration/custom-networking/custom_networking_test.go index 7cf9390453..94f29577a6 100644 --- a/test/integration/custom-networking/custom_networking_test.go +++ b/test/integration/custom-networking/custom_networking_test.go @@ -164,15 +164,11 @@ var _ = Describe("Custom Networking Test", func() { err = f.CloudServices.EC2().TerminateInstance(instanceIDs) Expect(err).ToNot(HaveOccurred()) - By("waiting for the node to be removed") + By("waiting for nodes to be removed") time.Sleep(time.Second * 120) - By("waiting for all nodes to become ready") - err = f.K8sResourceManagers.NodeManager(). - WaitTillNodesReady(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal, - nodeGroupProperties.AsgSize) - Expect(err).ToNot(HaveOccurred()) - + // Nodes should be stuck in NotReady state since no ENIs could be attached and no pod + // IP addresses are available. deployment := manifest.NewBusyBoxDeploymentBuilder(). Replicas(2). NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). diff --git a/testdata/amazon-eks-nodegroup.yaml b/testdata/amazon-eks-nodegroup.yaml index 89366e2bf8..1bcee36eb8 100644 --- a/testdata/amazon-eks-nodegroup.yaml +++ b/testdata/amazon-eks-nodegroup.yaml @@ -74,7 +74,6 @@ Parameters: NodeImageIdSSMParam: Type: "AWS::SSM::Parameter::Value" - Default: /aws/service/eks/optimized-ami/1.22/amazon-linux-2/recommended/image_id Description: AWS Systems Manager Parameter Store parameter of the AMI ID for the worker node instances. Change this value to match the version of Kubernetes you are using. DisableIMDSv1: