Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Testing time #76873

Merged
merged 5 commits into from
Jan 23, 2025
Merged

Testing time #76873

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
85c97ad
Testing time
jaredpar Jan 23, 2025
0457775
more
jaredpar Jan 23, 2025
a8ad58f
more
jaredpar Jan 23, 2025
802b448
more
jaredpar Jan 23, 2025
d88aae2
more
jaredpar Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 78 additions & 72 deletions src/Tools/Source/RunTests/AssemblyScheduler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Text.Json;
Expand Down Expand Up @@ -48,30 +50,48 @@ public static ImmutableArray<HelixWorkItem> Schedule(
{
// We didn't have any test history from azure devops, just partition by test count.
ConsoleUtil.Warning($"Could not look up test history - partitioning based on test count instead");
var workItemsByMethodCount = BuildWorkItems<int>(
var workItemsByMethodCount = BuildWorkItems(
orderedTypeInfos,
isOverLimitFunc: static (accumulatedMethodCount) => accumulatedMethodCount >= s_maxMethodCount,
addFunc: static (currentTest, accumulatedMethodCount) => accumulatedMethodCount + 1);
getWeightFunc: static test => 1,
limit: s_maxMethodCount);

LogWorkItems(workItemsByMethodCount);
return workItemsByMethodCount;
}

LogLongTests(testHistory);

// Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history
// so that we can extract an estimate of the test execution time for each test.
orderedTypeInfos = UpdateTestsWithExecutionTimes(orderedTypeInfos, testHistory);

// Create work items by partitioning tests by historical execution time with the goal of running under our time limit.
// While we do our best to run tests from the same assembly together (by building work items in assembly order) it is expected
// that some work items will run tests from multiple assemblies due to large variances in test execution time.
var workItems = BuildWorkItems<TimeSpan>(
var workItems = BuildWorkItems(
orderedTypeInfos,
isOverLimitFunc: static (accumulatedExecutionTime) => accumulatedExecutionTime >= s_maxExecutionTime,
addFunc: static (currentTest, accumulatedExecutionTime) => currentTest.ExecutionTime + accumulatedExecutionTime);
getWeightFunc: static test => test.ExecutionTime.TotalSeconds,
limit: s_maxExecutionTime.TotalSeconds);
LogWorkItems(workItems);
return workItems;
}

private static void LogLongTests(ImmutableDictionary<string, TimeSpan> testHistory)
{
var longTests = testHistory
.Where(kvp => kvp.Value > s_maxExecutionTime)
.OrderBy(kvp => kvp.Key)
.ToList();
if (longTests.Count > 0)
{
ConsoleUtil.Warning($"There are {longTests.Count} tests have execution times greater than the maximum execution time of {s_maxExecutionTime}");
foreach (var (test, time) in longTests)
{
ConsoleUtil.WriteLine($"\t{test} - {time:hh\\:mm\\:ss}");
}
}
}

private static ImmutableSortedDictionary<string, ImmutableArray<TypeInfo>> UpdateTestsWithExecutionTimes(
ImmutableSortedDictionary<string, ImmutableArray<TypeInfo>> assemblyTypes,
ImmutableDictionary<string, TimeSpan> testHistory)
Expand Down Expand Up @@ -137,105 +157,91 @@ void WriteResults()

private static ImmutableArray<HelixWorkItem> BuildWorkItems<TWeight>(
ImmutableSortedDictionary<string, ImmutableArray<TypeInfo>> typeInfos,
Func<TWeight, bool> isOverLimitFunc,
Func<TestMethodInfo, TWeight, TWeight> addFunc) where TWeight : struct
Func<TestMethodInfo, TWeight> getWeightFunc,
TWeight limit)
where TWeight : struct, INumber<TWeight>
{
var workItems = new List<HelixWorkItem>();
var currentWeight = TWeight.Zero;
var currentFilters = new List<(string AssemblyFilePath, TestMethodInfo TestMethodInfo)>();

// Keep track of the limit of the current work item we are adding to.
var accumulatedValue = default(TWeight);

// Keep track of the types we're planning to add to the current work item. The key
// is the file path of the assembly
var currentFilters = new SortedDictionary<string, List<TestMethodInfo>>();

// First find any assemblies we need to run in single assembly work items (due to state sharing concerns).
var singlePartitionAssemblies = typeInfos.Where(kvp => ShouldPartitionInSingleWorkItem(kvp.Key));
typeInfos = typeInfos.RemoveRange(singlePartitionAssemblies.Select(kvp => kvp.Key));
foreach (var (assemblyFilePaths, types) in singlePartitionAssemblies)
{
ConsoleUtil.WriteLine($"Building single assembly work item {workItems.Count} for {assemblyFilePaths}");
types.SelectMany(t => t.Tests).ToList().ForEach(test => AddFilter(assemblyFilePaths, test));

// End the work item so we don't include anything after this assembly.
AddCurrentWorkItem();
}

// Iterate through each assembly and type and build up the work items to run.
// We add types from assemblies one by one until we hit our limit,
// at which point we create a work item with the current types and start a new one.
foreach (var (assemblyFilePath, types) in typeInfos)
{
if (ShouldPartitionInSingleWorkItem(assemblyFilePath))
{
AddWorkItem(types.SelectMany(x => x.Tests).Select(x => (assemblyFilePath, x)));
continue;
}

foreach (var type in types)
{
foreach (var test in type.Tests)
{
// Get a new value representing the value from the test plus the accumulated value in the work item.
var newAccumulatedValue = addFunc(test, accumulatedValue);
var weight = getWeightFunc(test);

// When the single test is greater than the limit, give it a dedicated work item
if (weight > limit)
{
AddWorkItem([(assemblyFilePath, test)]);
continue;
}

currentWeight += weight;

// If the new accumulated value is greater than the limit
if (isOverLimitFunc(newAccumulatedValue))
// If the accumulated value is greater than the limit then we close off the current
// work item and start a new one
if (currentWeight > limit)
{
// Adding this type would put us over the time limit for this partition.
// Add the current work item to our list and start a new one.
AddCurrentWorkItem();
MaybeAddCurrentWorkItem();
currentWeight = weight;
}

// Update the current group in the work item with this new type.
AddFilter(assemblyFilePath, test);
currentFilters.Add((assemblyFilePath, test));
}
}
}

// Add any remaining tests to the work item.
AddCurrentWorkItem();
MaybeAddCurrentWorkItem();
return workItems.ToImmutableArray();

void AddCurrentWorkItem()
void MaybeAddCurrentWorkItem()
{
if (currentFilters.Any())
if (currentFilters.Count > 0)
{
var e = currentFilters.Values
.SelectMany(v => v)
.Sum(v => v.ExecutionTime.TotalSeconds);
var workItemInfo = new HelixWorkItem(
workItems.Count,
currentFilters.Keys.ToImmutableArray(),
currentFilters.Values.SelectMany(v => v).Select(x => x.FullyQualifiedName).ToImmutableArray(),
TimeSpan.FromSeconds(e));
workItems.Add(workItemInfo);
AddWorkItem(currentFilters);
currentFilters.Clear();
currentWeight = TWeight.Zero;
}

currentFilters.Clear();
accumulatedValue = default;
}

void AddFilter(string assemblyFilePath, TestMethodInfo test)
void AddWorkItem(params IEnumerable<(string AssemblyFilePath, TestMethodInfo TestMethodInfo)> tests)
{
if (!currentFilters.TryGetValue(assemblyFilePath, out var assemblyFilters))
{
assemblyFilters = new List<TestMethodInfo>();
currentFilters.Add(assemblyFilePath, assemblyFilters);
}

assemblyFilters.Add(test);
accumulatedValue = addFunc(test, accumulatedValue);
Debug.Assert(tests.Any());
var assemblyFilePaths = tests
.Select(x => x.AssemblyFilePath)
.Distinct()
.Order()
.ToImmutableArray();
var testMethodNames = tests
.Select(x => x.TestMethodInfo.FullyQualifiedName)
.ToImmutableArray();
var executionTime = tests
.Sum(x => x.TestMethodInfo.ExecutionTime.TotalSeconds);
var workItem = new HelixWorkItem(
workItems.Count,
assemblyFilePaths,
testMethodNames,
TimeSpan.FromSeconds(executionTime));
workItems.Add(workItem);
}
}

private static void LogWorkItems(ImmutableArray<HelixWorkItem> workItems)
{
ConsoleUtil.WriteLine($"Built {workItems.Length} work items");
ConsoleUtil.WriteLine("==== Work Item List ====");
foreach (var workItem in workItems)
{
ConsoleUtil.WriteLine($"- Work Item {workItem.Id} (Execution time {workItem.EstimatedExecutionTime})");
if (workItem.EstimatedExecutionTime > s_maxExecutionTime == true)
{
// Log a warning to the console with work item details when we were not able to partition in under our limit.
// This can happen when a single specific test exceeds our execution time limit.
ConsoleUtil.Warning($"Estimated execution {workItem.EstimatedExecutionTime} time exceeds max execution time {s_maxExecutionTime}.");
}
ConsoleUtil.WriteLine($"- Work Item: {workItem.Id} Execution time: {workItem.EstimatedExecutionTime:hh\\:mm\\:ss}");
}
}

Expand Down
Loading