diff --git a/plugins/ipam/dhcp/lease.go b/plugins/ipam/dhcp/lease.go
index 943e6e2c6..e36b768e5 100644
--- a/plugins/ipam/dhcp/lease.go
+++ b/plugins/ipam/dhcp/lease.go
@@ -455,7 +455,7 @@ func backoffRetry(resendMax time.Duration, f func() (*dhcp4.Packet, error)) (*dh
 		// only adjust delay time if we are in normal backoff stage
 		if baseDelay < resendMax && fastRetryLimit == 0 {
 			baseDelay *= 2
-		} else {
+		} else if fastRetryLimit == 0 {  // only break if we are at normal delay
 			break
 		}
 	}