Skip to content

Commit

Permalink
increase datastore pool at startup; custom networking optimization
Browse files Browse the repository at this point in the history
  • Loading branch information
@jdn5126
jdn5126 committed Apr 26, 2023
1 parent 62dbe0d commit 62bc27c
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 71 deletions.
14 changes: 8 additions & 6 deletions cmd/aws-vpc-cni/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
package main

import (
"bytes"
"encoding/json"
"net"
"os"
Expand Down Expand Up @@ -163,15 +162,18 @@ type Range struct {
Gateway net.IP `json:"gateway,omitempty"`
}

// Wait for IPAMD health check to pass (maximum wait time is 60 seconds)
func waitForIPAM() bool {
start := time.Now()
maxEnd := start.Add(time.Minute)
for {
cmd := exec.Command("./grpc-health-probe", "-addr", "127.0.0.1:50051", ">", "/dev/null", "2>&1")
var outb bytes.Buffer
cmd.Stdout = &outb
cmd.Run()
if outb.String() == "" {
if err := cmd.Run(); err == nil {
return true
}
if time.Now().After(maxEnd) {
return false
}
}
}

Expand Down Expand Up @@ -403,7 +405,7 @@ func _main() int {

err = cp.CopyFile(tmpAWSconflistFile, defaultHostCNIConfDirPath+awsConflistFile)
if err != nil {
log.WithError(err).Errorf("Failed to copy 10-awsconflist")
log.WithError(err).Errorf("Failed to copy %s", awsConflistFile)
return 1
}
log.Infof("Successfully copied CNI plugin binary and config file.")
Expand Down
2 changes: 1 addition & 1 deletion pkg/awsutils/awsutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string)

// New creates an EC2InstanceMetadataCache
func New(useCustomNetworking, disableENIProvisioning, v4Enabled, v6Enabled bool, eventRecorder *eventrecorder.EventRecorder) (*EC2InstanceMetadataCache, error) {
//ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run
// ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run
ctx := context.Background()

// Initializes prometheus metrics
Expand Down
25 changes: 12 additions & 13 deletions pkg/ipamd/datastore/data_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,13 +183,13 @@ type AddressInfo struct {

// CidrInfo
type CidrInfo struct {
//Either v4/v6 Host or LPM Prefix
// Either v4/v6 Host or LPM Prefix
Cidr net.IPNet
//Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6)
// Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6)
IPAddresses map[string]*AddressInfo
//true if Cidr here is an LPM prefix
// true if Cidr here is an LPM prefix
IsPrefix bool
//IP Address Family of the Cidr
// IP Address Family of the Cidr
AddressFamily string
}

Expand All @@ -199,8 +199,8 @@ func (cidr *CidrInfo) Size() int {
}

func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) {
//Either v4 or v6 for now.
//Check in V4 prefixes
// Either v4 or v6 for now.
// Check in V4 prefixes
for _, availableCidr := range e.AvailableIPv4Cidrs {
for _, addr := range availableCidr.IPAddresses {
if addr.IPAMKey == ipamKey {
Expand All @@ -209,7 +209,7 @@ func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) {
}
}

//Check in V6 prefixes
// Check in V6 prefixes
for _, availableCidr := range e.IPv6Cidrs {
for _, addr := range availableCidr.IPAddresses {
if addr.IPAMKey == ipamKey {
Expand Down Expand Up @@ -489,7 +489,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk,
ds.lock.Lock()
defer ds.lock.Unlock()

ds.log.Debugf("DataStore Add an ENI %s", eniID)
ds.log.Debugf("DataStore add an ENI %s", eniID)

_, ok := ds.eniPool[eniID]
if ok {
Expand Down Expand Up @@ -706,7 +706,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta
ds.lock.Lock()
defer ds.lock.Unlock()

ds.log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned)
ds.log.Debugf("AssignIPv4Address: IP address pool stats: total %d, assigned %d", ds.total, ds.assigned)

if eni, _, addr := ds.eniPool.FindAddressForSandbox(ipamKey); addr != nil {
ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for sandbox %s", ipamKey)
Expand Down Expand Up @@ -1117,8 +1117,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error {
func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) {
ds.lock.Lock()
defer ds.lock.Unlock()
ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total:%d, assigned %d, sandbox %s",
ds.total, ds.assigned, ipamKey)
ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total %d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey)

eni, availableCidr, addr := ds.eniPool.FindAddressForSandbox(ipamKey)
if addr == nil {
Expand Down Expand Up @@ -1235,7 +1234,7 @@ func (ds *DataStore) GetENIInfos() *ENIInfos {
tmpENIInfo := *eniInfo
tmpENIInfo.AvailableIPv4Cidrs = make(map[string]*CidrInfo, len(eniInfo.AvailableIPv4Cidrs))
tmpENIInfo.IPv6Cidrs = make(map[string]*CidrInfo, len(eniInfo.IPv6Cidrs))
for cidr, _ := range eniInfo.AvailableIPv4Cidrs {
for cidr := range eniInfo.AvailableIPv4Cidrs {
tmpENIInfo.AvailableIPv4Cidrs[cidr] = &CidrInfo{
Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr,
IPAddresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)),
Expand All @@ -1247,7 +1246,7 @@ func (ds *DataStore) GetENIInfos() *ENIInfos {
tmpENIInfo.AvailableIPv4Cidrs[cidr].IPAddresses[ip] = &ipAddrInfo
}
}
for cidr, _ := range eniInfo.IPv6Cidrs {
for cidr := range eniInfo.IPv6Cidrs {
tmpENIInfo.IPv6Cidrs[cidr] = &CidrInfo{
Cidr: eniInfo.IPv6Cidrs[cidr].Cidr,
IPAddresses: make(map[string]*AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)),
Expand Down
94 changes: 43 additions & 51 deletions pkg/ipamd/ipamd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,6 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex
c.enablePrefixDelegation = usePrefixDelegation()
c.enableIPv4 = isIPv4Enabled()
c.enableIPv6 = isIPv6Enabled()

c.disableENIProvisioning = disablingENIProvisioning()

client, err := awsutils.New(c.useCustomNetworking, c.disableENIProvisioning, c.enableIPv4, c.enableIPv6, c.eventRecorder)
Expand All @@ -411,7 +410,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex

c.primaryIP = make(map[string]string)
c.reconcileCooldownCache.cache = make(map[string]time.Time)
//WARM and Min IP/Prefix targets are ignored in IPv6 mode
// WARM and Min IP/Prefix targets are ignored in IPv6 mode
c.warmENITarget = getWarmENITarget()
c.warmIPTarget = getWarmIPTarget()
c.minimumIPTarget = getMinimumIPTarget()
Expand All @@ -427,8 +426,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex
return nil, err
}

//Let's validate if the configured combination of env variables is supported before we
//proceed any further
// Validate if the configured combination of env variables is supported before proceeding further
if !c.isConfigValid() {
return nil, fmt.Errorf("ipamd: failed to validate configuration")
}
Expand All @@ -445,7 +443,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex

mac := c.awsClient.GetPrimaryENImac()

// retrieve security groups
// Retrieve security groups
if c.enableIPv4 && !c.disableENIProvisioning {
err = c.awsClient.RefreshSGIDs(mac)
if err != nil {
Expand Down Expand Up @@ -475,16 +473,15 @@ func (c *IPAMContext) nodeInit() error {
}

if c.enableIPv4 {
//Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer
//be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled.
// Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer
// be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled.
vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs()
if err != nil {
return err
}
}

err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4,
c.enableIPv6)
err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4, c.enableIPv6)
if err != nil {
return errors.Wrap(err, "ipamd init: failed to set up host network")
}
Expand All @@ -501,7 +498,6 @@ func (c *IPAMContext) nodeInit() error {

for _, eni := range enis {
log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID)

isTrunkENI := eni.ENIID == metadataResult.TrunkENI
isEFAENI := metadataResult.EFAENIs[eni.ENIID]
if !isTrunkENI && !c.disableENIProvisioning {
Expand Down Expand Up @@ -541,22 +537,22 @@ func (c *IPAMContext) nodeInit() error {
}

if c.enableIPv6 {
//We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always
//start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation
//feature in IPv6 mode as we don't support (yet) a non-PD v6 option. In addition, we don't support custom
//networking & SGPP in IPv6 mode yet. So, we will skip the corresponding setup. Will save us from checking
//if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away
//with this check and not change anything else in the below setup.
// We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always
// start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation
// feature in IPv6 mode as we don't support (yet) a non-PD v6 option. In addition, we don't support custom
// networking & SGPP in IPv6 mode yet. So, we will skip the corresponding setup. Will save us from checking
// if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away
// with this check and not change anything else in the below setup.
return nil
}

if c.enablePrefixDelegation {
//During upgrade or if prefix delgation knob is disabled to enabled then we
//might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on
// During upgrade or if prefix delgation knob is disabled to enabled then we
// might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on
c.tryUnassignIPsFromENIs()
} else {
//When prefix delegation knob is enabled to disabled then we might
//have unused prefixes attached to the ENIs so need to cleanup
// When prefix delegation knob is enabled to disabled then we might
// have unused prefixes attached to the ENIs so need to cleanup
c.tryUnassignPrefixesFromENIs()
}

Expand Down Expand Up @@ -601,21 +597,18 @@ func (c *IPAMContext) nodeInit() error {
c.askForTrunkENIIfNeeded(ctx)
}

if !c.disableENIProvisioning {
// For a new node, attach Cidrs (secondary ips/prefixes)
increasedPool, err := c.tryAssignCidrs()
if err == nil && increasedPool {
c.updateLastNodeIPPoolAction()
} else if err != nil {
if containsInsufficientCIDRsOrSubnetIPs(err) {
log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation")
c.lastInsufficientCidrError = time.Now()
return nil
}
return err
// On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs.
if !c.disableENIProvisioning && c.isDatastorePoolTooLow() {
c.increaseDatastorePool(ctx)
// If custom networking is enabled and the pool is empty, return an error, as there is a misconfiguration and
// the node should not become ready.
if c.useCustomNetworking && c.isDatastorePoolEmpty() {
podENIErrInc("nodeInit")
return errors.New("Failed to attach any ENIs for custom networking")
}
}

log.Debug("node init completed successfully")
return nil
}

Expand Down Expand Up @@ -1117,9 +1110,9 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT
c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address()

if c.enableIPv6 && eni == primaryENI {
//In v6 PD Mode, VPC CNI will only manage primary ENI. Once we start supporting secondary IP and custom
//networking modes for v6, we will relax this restriction. We filter out all the ENIs except Primary ENI
//in v6 mode (prior to landing here), but included the primary ENI check as a safety net.
// In v6 PD Mode, VPC CNI will only manage primary ENI. Once we start supporting secondary IP and custom
// networking modes for v6, we will relax this restriction. We filter out all the ENIs except Primary ENI
// in v6 mode (prior to landing here), but included the primary ENI check as a safety net.
err := c.assignIPv6Prefix(eni)
if err != nil {
return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to Primary ENI")
Expand All @@ -1139,7 +1132,7 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT
}
}
log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes))
//Either case add the IPs and prefixes to datastore.
// Either case add the IPs and prefixes to datastore.
c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni)
c.addENIv4prefixesToDataStore(eniMetadata.IPv4Prefixes, eni)
}
Expand All @@ -1148,7 +1141,7 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT
}

func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []*ec2.NetworkInterfacePrivateIpAddress, eni string) {
//Add all the secondary IPs
// Add all the secondary IPs
for _, ec2PrivateIpAddr := range ec2PrivateIpAddrs {
if aws.BoolValue(ec2PrivateIpAddr.Primary) {
continue
Expand All @@ -1165,8 +1158,7 @@ func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []*ec2.Net
}

func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4PrefixSpecification, eni string) {

//Walk thru all prefixes
// Walk thru all prefixes
for _, ec2PrefixAddr := range ec2PrefixAddrs {
strIpv4Prefix := aws.StringValue(ec2PrefixAddr.Ipv4Prefix)
_, ipnet, err := net.ParseCIDR(strIpv4Prefix)
Expand All @@ -1188,12 +1180,12 @@ func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4Pref

func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv6PrefixSpecification, eni string) {
log.Debugf("Updating datastore with IPv6Prefix(es) for ENI: %v, count: %v", eni, len(ec2PrefixAddrs))
//Walk through all prefixes
// Walk through all prefixes
for _, ec2PrefixAddr := range ec2PrefixAddrs {
strIpv6Prefix := aws.StringValue(ec2PrefixAddr.Ipv6Prefix)
_, ipnet, err := net.ParseCIDR(strIpv6Prefix)
if err != nil {
//Parsing failed, get next prefix
// Parsing failed, get next prefix
log.Debugf("Parsing failed, moving on to next prefix")
continue
}
Expand Down Expand Up @@ -1832,7 +1824,6 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil
// accounting for the MINIMUM_IP_TARGET
// With prefix delegation this function determines the number of Prefixes `short` or `over`
func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) {

if c.warmIPTarget == noWarmIPTarget && c.minimumIPTarget == noMinimumIPTarget {
// there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI
return 0, 0, false
Expand All @@ -1854,9 +1845,8 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool)
over = max(min(over, stats.TotalIPs-c.minimumIPTarget), 0)

if c.enablePrefixDelegation {

//short : number of IPs short to reach warm targets
//over : number of IPs over the warm targets
// short : number of IPs short to reach warm targets
// over : number of IPs over the warm targets

_, numIPsPerPrefix, _ := datastore.GetPrefixDelegationDefaults()
// Number of prefixes IPAMD is short of to achieve warm targets
Expand All @@ -1876,7 +1866,6 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool)
overPrefix = max(min(overPrefix, stats.TotalPrefixes-prefixNeededForMinIP), 0)
log.Debugf("Current warm IP stats : target: %d, short(prefixes): %d, over(prefixes): %d, stats: %s", c.warmIPTarget, shortPrefix, overPrefix, stats)
return shortPrefix, overPrefix, true

}
log.Debugf("Current warm IP stats : target: %d, short: %d, over: %d, stats: %s", c.warmIPTarget, short, over, stats)

Expand Down Expand Up @@ -2197,6 +2186,11 @@ func (c *IPAMContext) GetIPv4Limit() (int, int, error) {
return maxIPsPerENI, maxPrefixesPerENI, nil
}

func (c *IPAMContext) isDatastorePoolEmpty() bool {
stats := c.dataStore.GetIPStats(ipV4AddrFamily)
return stats.TotalIPs == 0
}

func (c *IPAMContext) isDatastorePoolTooLow() bool {
short, _, warmTargetDefined := c.datastoreTargetState()
if warmTargetDefined {
Expand All @@ -2221,7 +2215,6 @@ func (c *IPAMContext) isDatastorePoolTooLow() bool {
c.logPoolStats(stats)
}
return poolTooLow

}

func (c *IPAMContext) isDatastorePoolTooHigh() bool {
Expand Down Expand Up @@ -2315,10 +2308,9 @@ func (c *IPAMContext) initENIAndIPLimits() (err error) {
log.Debugf("Max ip per ENI %d and max prefixes per ENI %d", c.maxIPsPerENI, c.maxPrefixesPerENI)
}

//WARM and MAX ENI & IP/Prefix counts are no-op in IPv6 Prefix delegation mode. Once we start supporting IPv6 in
//Secondary IP mode, these variables will play the same role as they currently do in IPv4 mode. So, for now we
//leave them at their default values.

// WARM and MAX ENI & IP/Prefix counts are no-op in IPv6 Prefix delegation mode. Once we start supporting IPv6 in
// Secondary IP mode, these variables will play the same role as they currently do in IPv4 mode. So, for now we
// leave them at their default values.
return nil
}

Expand Down

0 comments on commit 62bc27c

Please sign in to comment.