From 871c4e9fc8499a8f575af4f1bb7691138a98cdb0 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 13:32:39 -0700 Subject: [PATCH 1/6] retry quick query tests until we find a solution for closed stream problem. --- .../tests/BlobQuickQueryTests.cs | 18 +++++++++++++++++- .../tests/Shared/TestConstants.cs | 3 +++ .../tests/FileClientTests.cs | 12 ++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs index a13941d223397..da9b9055dfd3b 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs @@ -25,6 +25,7 @@ public BlobQuickQueryTests(bool async, BlobClientOptions.ServiceVersion serviceV [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Min() { // Arrange @@ -45,8 +46,8 @@ public async Task QueryAsync_Min() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Snapshot() { // Arrange @@ -70,6 +71,7 @@ public async Task QueryAsync_Snapshot() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Error() { // Arrange @@ -87,6 +89,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [Ignore("Don't want to record 16 MB of data.")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_MultipleDataRecords() { // Arrange @@ -129,6 +132,7 @@ public async Task QueryAsync_MultipleDataRecords() [RecordedTest] [Ignore("Don't want to record 250 MB of data.")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Large() { // Arrange @@ -162,6 +166,7 @@ public async Task QueryAsync_Large() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Progress() { // Arrange @@ -193,6 +198,7 @@ public async Task QueryAsync_Progress() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_QueryTextConfigurations() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -237,6 +243,7 @@ public async Task QueryAsync_QueryTextConfigurations() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_NonFatalError() { // Arrange @@ -277,6 +284,7 @@ public async Task QueryAsync_NonFatalError() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_FatalError() { // Arrange @@ -327,6 +335,7 @@ public async Task QueryAsync_FatalError() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); @@ -362,6 +371,7 @@ public async Task QueryAsync_AccessConditions() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); @@ -393,6 +403,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_IfTags() { // Arrange @@ -430,6 +441,7 @@ public async Task QueryAsync_IfTags() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_IfTags_Failed() { // Arrange @@ -458,6 +470,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ArrowConfiguration() { // Arrange @@ -498,6 +511,7 @@ public async Task QueryAsync_ArrowConfiguration() [Test] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ParquetConfiguration() { // Arrange @@ -528,6 +542,7 @@ public async Task QueryAsync_ParquetConfiguration() [Test] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ParquetOutputError() { // Arrange @@ -549,6 +564,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ArrowConfigurationInput() { // Arrange diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConstants.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConstants.cs index 81a2360bf0880..d68092b494ba6 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConstants.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConstants.cs @@ -23,6 +23,9 @@ public class TestConstants public const int DataLakeRetryDelay = 70000; public const int RetryDelay = 10000; + // quick query fails with with connection reset, retry until it's solved properly https://github.com/Azure/azure-sdk-for-net/issues/17403 + public const int QuickQueryRetryCount = 5; + public string CacheControl { get; private set; } public string ContentDisposition { get; private set; } public string ContentEncoding { get; private set; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index b41f1b19ecaf1..ced09bf775a90 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -3408,6 +3408,7 @@ public async Task ScheduleDeletionAsync_RemoveExpiry() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Min() { // Arrange @@ -3429,6 +3430,7 @@ public async Task QueryAsync_Min() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Error() { // Arrange @@ -3445,6 +3447,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_Progress() { // Arrange @@ -3476,6 +3479,7 @@ public async Task QueryAsync_Progress() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_QueryTextConfigurations() { await using DisposingFileSystem test = await GetNewFileSystem(); @@ -3520,6 +3524,7 @@ public async Task QueryAsync_QueryTextConfigurations() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_NonFatalError() { // Arrange @@ -3560,6 +3565,7 @@ public async Task QueryAsync_NonFatalError() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_FatalError() { // Arrange @@ -3608,6 +3614,7 @@ public async Task QueryAsync_FatalError() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); @@ -3644,6 +3651,7 @@ public async Task QueryAsync_AccessConditions() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); @@ -3675,6 +3683,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ArrowConfiguration() { // Arrange @@ -3714,6 +3723,7 @@ public async Task QueryAsync_ArrowConfiguration() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ArrowConfigurationInput() { // Arrange @@ -3751,6 +3761,7 @@ await TestHelper.AssertExpectedExceptionAsync( [Test] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ParquetConfigurationInput() { // Arrange @@ -3781,6 +3792,7 @@ public async Task QueryAsync_ParquetConfigurationInput() [Test] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] + [Retry(TestConstants.QuickQueryRetryCount)] public async Task QueryAsync_ParquetConfigurationOutputError() { // Arrange From 6f9a84a76196e927a1a7c652bf27c2f4899de29c Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 13:36:24 -0700 Subject: [PATCH 2/6] call some DFS apis as well. --- .../tests/DataLakeTestEnvironment.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs index 7bf503d48bfe2..6226847e1f46e 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.IO; using System.Threading.Tasks; using Azure.Storage.Test; using Azure.Storage.Test.Shared; @@ -38,6 +39,9 @@ private async Task DoesOAuthWorkAsync() var fileClient = directoryClient.GetFileClient(fileName); await fileClient.CreateIfNotExistsAsync(); await fileClient.GetPropertiesAsync(); + // call some APIs that talk to DFS endoint as well. + await fileClient.AppendAsync(new MemoryStream(new byte[] { 1 }), 0); + await fileClient.GetAccessControlAsync(); } finally { From 6a0e6f6c2ebaae38d44764385c4c54962f383dba Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 15:16:26 -0700 Subject: [PATCH 3/6] actually retry on exception... --- .../tests/BlobQuickQueryTests.cs | 35 +++---- .../tests/RetryOnExceptionAttributeTests.cs | 26 +++++ .../tests/Shared/RetryOnExceptionAttribute.cs | 99 +++++++++++++++++++ .../tests/FileClientTests.cs | 25 ++--- 4 files changed, 156 insertions(+), 29 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/tests/RetryOnExceptionAttributeTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/RetryOnExceptionAttribute.cs diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs index da9b9055dfd3b..b97c79cdda690 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobQuickQueryTests.cs @@ -12,6 +12,7 @@ using Azure.Storage.Blobs.Specialized; using Azure.Storage.Test; using Azure.Storage.Test.Shared; +using Azure.Storage.Tests.Shared; using NUnit.Framework; namespace Azure.Storage.Blobs.Test @@ -25,7 +26,7 @@ public BlobQuickQueryTests(bool async, BlobClientOptions.ServiceVersion serviceV [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Min() { // Arrange @@ -47,7 +48,7 @@ public async Task QueryAsync_Min() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Snapshot() { // Arrange @@ -71,7 +72,7 @@ public async Task QueryAsync_Snapshot() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Error() { // Arrange @@ -89,7 +90,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [Ignore("Don't want to record 16 MB of data.")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_MultipleDataRecords() { // Arrange @@ -132,7 +133,7 @@ public async Task QueryAsync_MultipleDataRecords() [RecordedTest] [Ignore("Don't want to record 250 MB of data.")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Large() { // Arrange @@ -166,7 +167,7 @@ public async Task QueryAsync_Large() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Progress() { // Arrange @@ -198,7 +199,7 @@ public async Task QueryAsync_Progress() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_QueryTextConfigurations() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -243,7 +244,7 @@ public async Task QueryAsync_QueryTextConfigurations() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_NonFatalError() { // Arrange @@ -284,7 +285,7 @@ public async Task QueryAsync_NonFatalError() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_FatalError() { // Arrange @@ -335,7 +336,7 @@ public async Task QueryAsync_FatalError() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); @@ -371,7 +372,7 @@ public async Task QueryAsync_AccessConditions() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); @@ -403,7 +404,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_IfTags() { // Arrange @@ -441,7 +442,7 @@ public async Task QueryAsync_IfTags() [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_IfTags_Failed() { // Arrange @@ -470,7 +471,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ArrowConfiguration() { // Arrange @@ -511,7 +512,7 @@ public async Task QueryAsync_ArrowConfiguration() [Test] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ParquetConfiguration() { // Arrange @@ -542,7 +543,7 @@ public async Task QueryAsync_ParquetConfiguration() [Test] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ParquetOutputError() { // Arrange @@ -564,7 +565,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ArrowConfigurationInput() { // Arrange diff --git a/sdk/storage/Azure.Storage.Common/tests/RetryOnExceptionAttributeTests.cs b/sdk/storage/Azure.Storage.Common/tests/RetryOnExceptionAttributeTests.cs new file mode 100644 index 0000000000000..2716f050299b0 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/RetryOnExceptionAttributeTests.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Storage.Tests.Shared; +using NUnit.Framework; + +namespace Azure.Storage.Tests +{ + public class RetryOnExceptionAttributeTests + { + private int counter = 0; + + [Test] + [RetryOnException(3, typeof(InvalidOperationException))] + public void ShouldRetryOnException() + { + if (counter++ == 0) + { + throw new InvalidOperationException(); + } + + Assert.AreEqual(2, counter); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/RetryOnExceptionAttribute.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/RetryOnExceptionAttribute.cs new file mode 100644 index 0000000000000..eddb4b86ca82e --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/RetryOnExceptionAttribute.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; +using NUnit.Framework.Internal.Commands; + +namespace Azure.Storage.Tests.Shared +{ + /// + /// This is clone of Retry from NUnit. Except it retries on exception. NUnit version retries on assertion error only. + /// Use only to deal with flaky live tests. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = false)] + public class RetryOnExceptionAttribute : NUnitAttribute, IRepeatTest + { + private readonly int _tryCount; + private readonly Type _exceptionType; + + public RetryOnExceptionAttribute(int tryCount, Type exceptionType) + { + _tryCount = tryCount; + _exceptionType = exceptionType; + } + + #region IRepeatTest Members + public TestCommand Wrap(TestCommand command) + { + return new RetryCommand(command, _tryCount, _exceptionType); + } + + #endregion + + #region Nested RetryCommand Class + + public class RetryCommand : DelegatingTestCommand + { + private readonly int _tryCount; + private readonly Type _exceptionType; + + public RetryCommand(TestCommand innerCommand, int tryCount, Type exceptionType) + : base(innerCommand) + { + _tryCount = tryCount; + _exceptionType = exceptionType; + } + + public override TestResult Execute(TestExecutionContext context) + { + int count = _tryCount; + + while (count-- > 0) + { + try + { + context.CurrentResult = innerCommand.Execute(context); + } + // Commands are supposed to catch exceptions, but some don't + // and we want to look at restructuring the API in the future. + catch (Exception ex) + { + if (context.CurrentResult == null) + context.CurrentResult = context.CurrentTest.MakeTestResult(); + context.CurrentResult.RecordException(ex); + } + + // Clear result for retry + if (count > 0 && IsTestFailedWithExpectedException(context)) + { + context.CurrentResult = context.CurrentTest.MakeTestResult(); + context.CurrentRepeatCount++; // increment Retry count for next iteration. will only happen if we are guaranteed another iteration + } + else + { + break; + } + } + + return context.CurrentResult; + } + + private bool IsTestFailedWithExpectedException(TestExecutionContext context) + { + var failed = context.CurrentResult.ResultState.Status switch + { + TestStatus.Passed => false, + TestStatus.Skipped => false, + _ => true + }; + + return failed && context.CurrentResult.Message.Contains(_exceptionType.FullName); + } + } + + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index ced09bf775a90..0e6f3f777efde 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -16,6 +16,7 @@ using Azure.Storage.Files.DataLake.Models; using Azure.Storage.Sas; using Azure.Storage.Test; +using Azure.Storage.Tests.Shared; using Moq; using NUnit.Framework; @@ -3408,7 +3409,7 @@ public async Task ScheduleDeletionAsync_RemoveExpiry() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Min() { // Arrange @@ -3430,7 +3431,7 @@ public async Task QueryAsync_Min() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Error() { // Arrange @@ -3447,7 +3448,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_Progress() { // Arrange @@ -3479,7 +3480,7 @@ public async Task QueryAsync_Progress() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_QueryTextConfigurations() { await using DisposingFileSystem test = await GetNewFileSystem(); @@ -3524,7 +3525,7 @@ public async Task QueryAsync_QueryTextConfigurations() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_NonFatalError() { // Arrange @@ -3565,7 +3566,7 @@ public async Task QueryAsync_NonFatalError() [RecordedTest] [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/12063")] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_FatalError() { // Arrange @@ -3614,7 +3615,7 @@ public async Task QueryAsync_FatalError() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); @@ -3651,7 +3652,7 @@ public async Task QueryAsync_AccessConditions() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); @@ -3683,7 +3684,7 @@ await TestHelper.AssertExpectedExceptionAsync( [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ArrowConfiguration() { // Arrange @@ -3723,7 +3724,7 @@ public async Task QueryAsync_ArrowConfiguration() [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ArrowConfigurationInput() { // Arrange @@ -3761,7 +3762,7 @@ await TestHelper.AssertExpectedExceptionAsync( [Test] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ParquetConfigurationInput() { // Arrange @@ -3792,7 +3793,7 @@ public async Task QueryAsync_ParquetConfigurationInput() [Test] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_08_04)] [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/19575")] - [Retry(TestConstants.QuickQueryRetryCount)] + [RetryOnException(TestConstants.QuickQueryRetryCount, typeof(IOException))] public async Task QueryAsync_ParquetConfigurationOutputError() { // Arrange From 215311435d2dd6d34244bbf31786c1f8185ab605 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 15:18:36 -0700 Subject: [PATCH 4/6] log to error to show up in the logs.. --- .../Azure.Storage.Blobs/tests/BlobTestEnvironment.cs | 6 +++--- .../tests/DataLakeTestEnvironment.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs index 5bd212f3eaf69..bf20261fae951 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs @@ -19,7 +19,7 @@ protected override async ValueTask IsEnvironmentReadyAsync() private async Task DoesOAuthWorkAsync() { - TestContext.Out.WriteLine("Blob Probing OAuth"); + TestContext.Error.WriteLine("Blob Probing OAuth"); BlobServiceClient serviceClient = new BlobServiceClient( new Uri(TestConfigurations.DefaultTargetOAuthTenant.BlobServiceEndpoint), GetOAuthCredential(TestConfigurations.DefaultTargetOAuthTenant)); @@ -43,10 +43,10 @@ private async Task DoesOAuthWorkAsync() } } catch (RequestFailedException e) when (e.Status == 403 && e.ErrorCode == "AuthorizationPermissionMismatch") { - TestContext.Out.WriteLine("Blob Probing OAuth - not ready"); + TestContext.Error.WriteLine("Blob Probing OAuth - not ready"); return false; } - TestContext.Out.WriteLine("Blob Probing OAuth - ready"); + TestContext.Error.WriteLine("Blob Probing OAuth - ready"); return true; } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs index 6226847e1f46e..79e38ae17b18d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs @@ -19,7 +19,7 @@ protected override async ValueTask IsEnvironmentReadyAsync() private async Task DoesOAuthWorkAsync() { - TestContext.Out.WriteLine("Datalake Probing OAuth"); + TestContext.Error.WriteLine("Datalake Probing OAuth"); DataLakeServiceClient serviceClient = new DataLakeServiceClient( new Uri(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant.BlobServiceEndpoint), GetOAuthCredential(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant)); @@ -50,10 +50,10 @@ private async Task DoesOAuthWorkAsync() } catch (RequestFailedException e) when (e.Status == 403 && e.ErrorCode == "AuthorizationPermissionMismatch") { - TestContext.Out.WriteLine("Datalake Probing OAuth - not ready"); + TestContext.Error.WriteLine("Datalake Probing OAuth - not ready"); return false; } - TestContext.Out.WriteLine("Datalake Probing OAuth - ready"); + TestContext.Error.WriteLine("Datalake Probing OAuth - ready"); return true; } } From e6ba564c84296c36c78f2d474cb1c4041442bde4 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 16:08:43 -0700 Subject: [PATCH 5/6] these role assignments are on RG level. --- sdk/storage/test-resources.json | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/sdk/storage/test-resources.json b/sdk/storage/test-resources.json index 314f82506bf2d..6f86fa3ad02b6 100644 --- a/sdk/storage/test-resources.json +++ b/sdk/storage/test-resources.json @@ -60,10 +60,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "[variables('authorizationApiVersion')]", - "name": "[guid(concat('dataContributorRoleId', variables('primaryAccountName')))]", - "dependsOn": [ - "[variables('primaryAccountName')]" - ], + "name": "[guid(concat('dataContributorRoleId', resourceGroup().id))]", "properties": { "roleDefinitionId": "[variables('blobDataContributorRoleId')]", "principalId": "[parameters('testApplicationOid')]" @@ -72,10 +69,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "[variables('authorizationApiVersion')]", - "name": "[guid(concat('contributorRoleId', variables('primaryAccountName')))]", - "dependsOn": [ - "[variables('primaryAccountName')]" - ], + "name": "[guid(concat('contributorRoleId', resourceGroup().id))]", "properties": { "roleDefinitionId": "[variables('contributorRoleId')]", "principalId": "[parameters('testApplicationOid')]" @@ -84,10 +78,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "[variables('authorizationApiVersion')]", - "name": "[guid(concat('blobDataOwnerRoleId', variables('primaryAccountName')))]", - "dependsOn": [ - "[variables('primaryAccountName')]" - ], + "name": "[guid(concat('blobDataOwnerRoleId', resourceGroup().id))]", "properties": { "roleDefinitionId": "[variables('blobDataOwnerRoleId')]", "principalId": "[parameters('testApplicationOid')]" From 34ca34a97edf6af981f1256e85213f4580aaa3ba Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Thu, 22 Apr 2021 16:37:55 -0700 Subject: [PATCH 6/6] tweaks. --- .../tests/BlobTestEnvironment.cs | 7 +- .../tests/DataLakeTestEnvironment.cs | 71 +++++++++++++------ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs index bf20261fae951..77f89eaf1b835 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestEnvironment.cs @@ -20,11 +20,12 @@ protected override async ValueTask IsEnvironmentReadyAsync() private async Task DoesOAuthWorkAsync() { TestContext.Error.WriteLine("Blob Probing OAuth"); - BlobServiceClient serviceClient = new BlobServiceClient( - new Uri(TestConfigurations.DefaultTargetOAuthTenant.BlobServiceEndpoint), - GetOAuthCredential(TestConfigurations.DefaultTargetOAuthTenant)); + try { + BlobServiceClient serviceClient = new BlobServiceClient( + new Uri(TestConfigurations.DefaultTargetOAuthTenant.BlobServiceEndpoint), + GetOAuthCredential(TestConfigurations.DefaultTargetOAuthTenant)); await serviceClient.GetPropertiesAsync(); var containerName = Guid.NewGuid().ToString(); var containerClient = serviceClient.GetBlobContainerClient(containerName); diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs index 79e38ae17b18d..1eb4b58463f1e 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeTestEnvironment.cs @@ -4,6 +4,8 @@ using System; using System.IO; using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; using Azure.Storage.Test; using Azure.Storage.Test.Shared; using NUnit.Framework; @@ -20,32 +22,59 @@ protected override async ValueTask IsEnvironmentReadyAsync() private async Task DoesOAuthWorkAsync() { TestContext.Error.WriteLine("Datalake Probing OAuth"); - DataLakeServiceClient serviceClient = new DataLakeServiceClient( - new Uri(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant.BlobServiceEndpoint), - GetOAuthCredential(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant)); + try { - await serviceClient.GetPropertiesAsync(); - var fileSystemName = Guid.NewGuid().ToString(); - var fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName); - await fileSystemClient.CreateIfNotExistsAsync(); - try + // Check flat account. For some reason we observe failures if that one doesn't work before we start datalake run. { - var directoryName = Guid.NewGuid().ToString(); - var directoryClient = fileSystemClient.GetDirectoryClient(directoryName); - await directoryClient.CreateIfNotExistsAsync(); - await directoryClient.GetPropertiesAsync(); - var fileName = Guid.NewGuid().ToString(); - var fileClient = directoryClient.GetFileClient(fileName); - await fileClient.CreateIfNotExistsAsync(); - await fileClient.GetPropertiesAsync(); - // call some APIs that talk to DFS endoint as well. - await fileClient.AppendAsync(new MemoryStream(new byte[] { 1 }), 0); - await fileClient.GetAccessControlAsync(); + BlobServiceClient serviceClient = new BlobServiceClient( + new Uri(TestConfigurations.DefaultTargetOAuthTenant.BlobServiceEndpoint), + GetOAuthCredential(TestConfigurations.DefaultTargetOAuthTenant)); + await serviceClient.GetPropertiesAsync(); + var containerName = Guid.NewGuid().ToString(); + var containerClient = serviceClient.GetBlobContainerClient(containerName); + await containerClient.CreateIfNotExistsAsync(); + try + { + await containerClient.GetPropertiesAsync(); + var blobName = Guid.NewGuid().ToString(); + var blobClient = containerClient.GetAppendBlobClient(blobName); + await blobClient.CreateIfNotExistsAsync(); + await blobClient.GetPropertiesAsync(); + } + finally + { + await containerClient.DeleteIfExistsAsync(); + } } - finally + + // Check hierarchical account. { - await fileSystemClient.DeleteIfExistsAsync(); + DataLakeServiceClient serviceClient = new DataLakeServiceClient( + new Uri(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant.BlobServiceEndpoint), + GetOAuthCredential(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant)); + await serviceClient.GetPropertiesAsync(); + var fileSystemName = Guid.NewGuid().ToString(); + var fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName); + await fileSystemClient.CreateIfNotExistsAsync(); + try + { + var directoryName = Guid.NewGuid().ToString(); + var directoryClient = fileSystemClient.GetDirectoryClient(directoryName); + await directoryClient.CreateIfNotExistsAsync(); + await directoryClient.GetPropertiesAsync(); + var fileName = Guid.NewGuid().ToString(); + var fileClient = directoryClient.GetFileClient(fileName); + await fileClient.CreateIfNotExistsAsync(); + await fileClient.GetPropertiesAsync(); + // call some APIs that talk to DFS endoint as well. + await fileClient.AppendAsync(new MemoryStream(new byte[] { 1 }), 0); + await fileClient.GetAccessControlAsync(); + } + finally + { + await fileSystemClient.DeleteIfExistsAsync(); + } } } catch (RequestFailedException e) when (e.Status == 403 && e.ErrorCode == "AuthorizationPermissionMismatch")