Skip to content

Commit

Permalink
prevent livelock when increasing datastore pool
Browse files Browse the repository at this point in the history
  • Loading branch information
jdn5126 committed Feb 26, 2024
1 parent 315d56a commit 2e74b1e
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions pkg/ipamd/ipamd.go
Original file line number Diff line number Diff line change
Expand Up @@ -659,11 +659,13 @@ func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) {
// Each iteration, log the current datastore IP stats
log.Debugf("IP stats - total IPs: %d, assigned IPs: %d, cooldown IPs: %d", stats.TotalIPs, stats.AssignedIPs, stats.CooldownIPs)

// Scale up rapidly to decrease the time it takes for new pods to get an IP assigned. Scale down conservatively to account for pod churn.
if datastorePoolTooLow {
// Allow for rapid scale up to decrease time it takes for pod to retrieve an ip
// but conservative scale down to account for pod churn
for datastorePoolStillTooLow := datastorePoolTooLow; datastorePoolStillTooLow; datastorePoolStillTooLow, _ = c.isDatastorePoolTooLow() {
// Prevent livelock by limiting the number of attempts at scaling up.
count := 0
for datastorePoolStillTooLow := datastorePoolTooLow; datastorePoolStillTooLow && count < 3; datastorePoolStillTooLow, _ = c.isDatastorePoolTooLow() {
c.increaseDatastorePool(ctx)
count += 1
}
} else if c.isDatastorePoolTooHigh(stats) {
c.decreaseDatastorePool(decreaseIPPoolInterval)
Expand Down

0 comments on commit 2e74b1e

Please sign in to comment.