Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Retry http 429 responses #4897

Merged
merged 3 commits into from
Nov 10, 2022
Merged

Retry http 429 responses #4897

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f07e846
Add test to validate HTTP 429 retries
zivkan Nov 4, 2022
c9a94bb
Retry on HTTP 429 and 408
zivkan Nov 4, 2022
88f0e58
Add EnvVar to disable 429 retries
zivkan Nov 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/NuGet.Core/NuGet.Protocol/HttpSource/HttpRetryHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,11 @@ public async Task<HttpResponseMessage> SendAsync(
requestUri,
bodyStopwatch.ElapsedMilliseconds));

if ((int)response.StatusCode >= 500)
int statusCode = (int)response.StatusCode;
// 5xx == server side failure
// 408 == request timeout
// 429 == too many requests
if (statusCode >= 500 || statusCode == 408 || statusCode == 429)
{
success = false;
}
Expand Down
105 changes: 49 additions & 56 deletions test/NuGet.Core.Tests/NuGet.Protocol.Tests/HttpSource/HttpRetryHandlerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ namespace NuGet.Protocol.Tests
public class HttpRetryHandlerTests
{
private const int MaxTries = 5;
private const string TestUrl = "https://test.local/test.json";
private const string TestUrl = "https://local.test/test.json";
private static readonly TimeSpan SmallTimeout = TimeSpan.FromMilliseconds(50);
private static readonly TimeSpan LargeTimeout = TimeSpan.FromSeconds(5);

Expand Down Expand Up @@ -100,12 +100,7 @@ public async Task HttpRetryHandler_DifferentRequestInstanceEachTime()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();

var requests = new HashSet<HttpRequestMessage>();
Func<HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
Expand Down Expand Up @@ -140,12 +135,7 @@ public async Task HttpRetryHandler_CancelsRequestAfterTimeout()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
var requestToken = CancellationToken.None;
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> handler = async (requestMessage, token) =>
{
Expand Down Expand Up @@ -182,12 +172,7 @@ public async Task HttpRetryHandler_ThrowsTimeoutExceptionForTimeout()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> handler = async (requestMessage, token) =>
{
await Task.Delay(LargeTimeout);
Expand Down Expand Up @@ -220,12 +205,7 @@ public async Task HttpRetryHandler_MultipleTriesTimed()
// Arrange
TimeSpan retryDelay = SmallTimeout;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
Func<HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
return new HttpResponseMessage(HttpStatusCode.ServiceUnavailable);
Expand Down Expand Up @@ -266,12 +246,7 @@ public async Task HttpRetryHandler_MultipleTriesNoSuccess()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
var hits = 0;

Func<HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
Expand Down Expand Up @@ -307,12 +282,7 @@ public async Task HttpRetryHandler_MultipleSuccessFirstVerifySingleHit()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
var hits = 0;

Func<HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
Expand Down Expand Up @@ -347,12 +317,7 @@ public async Task HttpRetryHandler_404VerifySingleHit()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
var hits = 0;

Func<HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
Expand Down Expand Up @@ -432,12 +397,7 @@ public async Task HttpRetryHandler_MultipleTriesUntilSuccess()
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables();
var tries = 0;
var sent503 = false;

Expand Down Expand Up @@ -501,13 +461,7 @@ public async Task HttpRetryHandler_EnhancedRetryAllowsSettingMoreRetries()
}
};

TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.IsEnabledEnvironmentVariableName] = bool.TrueString,
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = "11",
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = "3"
});
TestEnvironmentVariableReader testEnvironmentVariableReader = GetEnhancedHttpRetryEnvironmentVariables(retryCount: 11, delayMilliseconds: 3);

EnhancedHttpRetryHelper helper = new EnhancedHttpRetryHelper(testEnvironmentVariableReader);
Assert.Equal(helper.IsEnabled, true);
Expand Down Expand Up @@ -564,11 +518,50 @@ public void HttpRetryHandler_EnhancedRetryOnByDefault(string value, bool expecte
Assert.Equal(helper.DelayInMilliseconds, EnhancedHttpRetryHelper.DefaultDelayMilliseconds);
}

[Fact]
public async Task HttpRetryHandler_RetriesOn429()
{
// Arrange
int attempt = 0;
// .NET Framework don't have HttpStatusCode.TooManyRequests
HttpStatusCode tooManyRequestsStatusCode = (HttpStatusCode)429;

var busyServer = new HttpRetryTestHandler((req, cancellationToken) =>
attempt++ < 2
? Task.FromResult(new HttpResponseMessage(tooManyRequestsStatusCode))
: Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var client = new HttpClient(busyServer);

var httpRetryHandlerRequest = new HttpRetryHandlerRequest(client, () => new HttpRequestMessage(HttpMethod.Get, TestUrl));
TestEnvironmentVariableReader environmentVariables = GetEnhancedHttpRetryEnvironmentVariables();
var httpRetryHandler = new HttpRetryHandler(environmentVariables);
var logger = new TestLogger();

// Act
var response = await httpRetryHandler.SendAsync(httpRetryHandlerRequest, logger, CancellationToken.None);

// Assert
response.StatusCode.Should().Be(HttpStatusCode.OK);
attempt.Should().Be(3);

string expectedMessagePrefix = " " + tooManyRequestsStatusCode.ToString() + " " + TestUrl;
logger.Messages.Where(m => m.StartsWith(expectedMessagePrefix, StringComparison.Ordinal)).Count().Should().Be(2);
}

private static TimeSpan GetRetryMinTime(int tries, TimeSpan retryDelay)
{
return TimeSpan.FromTicks((tries - 1) * retryDelay.Ticks);
}

private static TestEnvironmentVariableReader GetEnhancedHttpRetryEnvironmentVariables(bool? isEnabled = true, int? retryCount = MaxTries, int? delayMilliseconds = 0)
=> new TestEnvironmentVariableReader(
new Dictionary<string, string>()
{
[EnhancedHttpRetryHelper.IsEnabledEnvironmentVariableName] = isEnabled?.ToString(),
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = retryCount?.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = delayMilliseconds?.ToString()
});

private class TestHandler : DelegatingHandler
{
public HttpRequestMessage LastRequest { get; private set; }
Expand Down