-
Notifications
You must be signed in to change notification settings - Fork 741
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add FlakyAttribute to mark flaky tests
part of dotnet/aspnetcore#8237
- Loading branch information
1 parent
11727db
commit 15e5ac6
Showing
4 changed files
with
168 additions
and
0 deletions.
There are no files selected for viewing
28 changes: 28 additions & 0 deletions
28
src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Reflection; | ||
|
||
namespace Microsoft.AspNetCore.Testing | ||
{ | ||
public static class HelixQueues | ||
{ | ||
public const string All = "all"; | ||
public const string None = "none"; | ||
|
||
// Queue names end in ';' because it makes it easier to concat these into a list using a constant expression: | ||
// HelixQueues.Fedora28 + HelixQueues.Centos7 | ||
|
||
public const string Fedora28 = "Fedora.28." + HelixSuffix + ";"; | ||
public const string Fedora27 = "Fedora.27." + HelixSuffix + ";"; | ||
public const string Redhat7 = "Redhat.7." + HelixSuffix + ";"; | ||
public const string Debian9 = "Debian.9." + HelixSuffix + ";"; | ||
public const string Debian8 = "Debian.8." + HelixSuffix + ";"; | ||
public const string Centos7 = "Centos.7." + HelixSuffix + ";"; | ||
public const string Ubuntu1604 = "Ubuntu.1604." + HelixSuffix + ";"; | ||
public const string Ubuntu1810 = "Ubuntu.1810." + HelixSuffix + ";"; | ||
public const string macOS1012 = "OSX.1012." + HelixSuffix + ";"; | ||
public const string Windows10 = "Windows.10.Amd64.ClientRS4.VS2017.Open;"; // Doesn't have the default suffix! | ||
|
||
private const string HelixSuffix = "Amd64.Open"; | ||
} | ||
} |
43 changes: 43 additions & 0 deletions
43
src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using Xunit.Sdk; | ||
|
||
namespace Microsoft.AspNetCore.Testing.xunit | ||
{ | ||
[TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")] | ||
[AttributeUsage(AttributeTargets.Method)] | ||
public sealed class FlakyAttribute : Attribute, ITraitAttribute | ||
{ | ||
private List<string> _queues = new List<string>() { "all" }; | ||
|
||
/// <summary> | ||
/// Gets a URL to a GitHub issue tracking this flaky test. | ||
/// </summary> | ||
public string GitHubIssueUrl { get; } | ||
|
||
/// <summary> | ||
/// Gets or sets a list of helix queues on which this test is flaky. Defaults to <see cref="HelixQueues.All"/> indicating it is flaky on all Helix queues. See | ||
/// <see cref="HelixQueues"/> for a list of valid values. | ||
/// </summary> | ||
public string OnHelixQueues | ||
{ | ||
get => string.Join(";", _queues); | ||
set => _queues = new List<string>(value.Split(';')); | ||
} | ||
|
||
/// <summary> | ||
/// Gets or sets a boolean indicating if this test is flaky on Azure DevOps Pipelines. Defaults to <see langword="true" />. | ||
/// </summary> | ||
public bool OnAzDO { get; set; } = true; | ||
|
||
/// <summary> | ||
/// Gets a list of Helix queues on which the test is flaky (including <see cref="HelixQueues.All"/> if specified). | ||
/// </summary> | ||
public IReadOnlyList<string> FlakyQueues => _queues; | ||
|
||
public FlakyAttribute(string gitHubIssueUrl) | ||
{ | ||
GitHubIssueUrl = gitHubIssueUrl; | ||
} | ||
} | ||
} |
35 changes: 35 additions & 0 deletions
35
src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using Xunit.Abstractions; | ||
using Xunit.Sdk; | ||
|
||
namespace Microsoft.AspNetCore.Testing.xunit | ||
{ | ||
public class FlakyTestDiscoverer : ITraitDiscoverer | ||
{ | ||
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute) | ||
{ | ||
if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute) | ||
{ | ||
return GetTraitsCore(flakyAttribute); | ||
} | ||
else | ||
{ | ||
throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection."); | ||
} | ||
} | ||
|
||
private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute) | ||
{ | ||
if(attribute.OnAzDO) | ||
{ | ||
yield return new KeyValuePair<string, string>("Flaky:AzDO", "true"); | ||
} | ||
|
||
foreach(var queue in attribute.FlakyQueues) | ||
{ | ||
yield return new KeyValuePair<string, string>($"Flaky:Helix:{queue}", "true"); | ||
} | ||
} | ||
} | ||
} |
62 changes: 62 additions & 0 deletions
62
src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
using Microsoft.AspNetCore.Testing.xunit; | ||
using System; | ||
using System.Collections.Generic; | ||
using Xunit; | ||
|
||
namespace Microsoft.AspNetCore.Testing.Tests | ||
{ | ||
public class FlakyAttributeTest | ||
{ | ||
[Fact] | ||
[Flaky("http://example.com")] | ||
public void AlwaysFlaky() | ||
{ | ||
throw new Exception("Flakey!"); | ||
} | ||
|
||
[Fact] | ||
[Flaky("http://example.com", OnAzDO = false)] | ||
public void FlakyInHelixOnly() | ||
{ | ||
// TODO: Use actual Helix detection variable ;) | ||
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX"))) | ||
{ | ||
throw new Exception("Flaky on Helix!"); | ||
} | ||
} | ||
|
||
[Fact] | ||
[Flaky("http://example.com", OnAzDO = false, OnHelixQueues = HelixQueues.macOS1012 + HelixQueues.Fedora28)] | ||
public void FlakyInSpecificHelixQueue() | ||
{ | ||
// TODO: Use actual Helix detection variable ;) | ||
var queueName = Environment.GetEnvironmentVariable("HELIX"); | ||
if (!string.IsNullOrEmpty(queueName)) | ||
{ | ||
|
||
// Normalize the queue name to have a trailing ';' (this is only for testing anyway) | ||
if (!queueName.EndsWith(";")) | ||
{ | ||
queueName = $"{queueName};"; | ||
} | ||
|
||
var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012, HelixQueues.Fedora28 }; | ||
if (failingQueues.Contains(queueName)) | ||
{ | ||
throw new Exception("Flaky on Helix!"); | ||
} | ||
} | ||
} | ||
|
||
[Fact] | ||
[Flaky("http://example.com", OnHelixQueues = HelixQueues.None)] | ||
public void FlakyInAzDoOnly() | ||
{ | ||
// TODO: Use actual AzDO detection variable ;) | ||
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AZDO"))) | ||
{ | ||
throw new Exception("Flaky on AzDO!"); | ||
} | ||
} | ||
} | ||
} |