Skip to content

Commit

Permalink
Fix scaling issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@jaredpar
jaredpar committed Dec 1, 2023
1 parent 7374087 commit eeb9405
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 28 deletions.
2 changes: 1 addition & 1 deletion .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/scratch/bin/Debug/netcoreapp3.1/scratch.dll",
"program": "${workspaceFolder}/scratch/bin/Debug/net6.0/scratch.dll",
"args": [
],
"cwd": "${workspaceFolder}/scratch",
Expand Down
62 changes: 41 additions & 21 deletions DevOps.Util.DotNet/Function/FunctionUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,13 @@ public async Task OnPullRequestMergedAsync(
/// </summary>
public async Task<int> DeleteOldBuilds(TriageContext triageContext, int deleteMax = 25)
{
var limitDays = 30;
var limitDays = 21;
var limit = DateTime.UtcNow - TimeSpan.FromDays(limitDays);
var count = 0;

var builds = await triageContext
.ModelBuilds
.AsNoTracking()
.Where(x => x.StartTime < limit)
.OrderBy(x => x.StartTime)
.Take(deleteMax)
Expand Down Expand Up @@ -96,24 +97,40 @@ public async Task<int> DeleteOldBuilds(TriageContext triageContext, int deleteMa

async Task DeleteTimelineIssues(ModelBuild modelBuild)
{
var count = await triageContext
var max = 100;
var totalCount = await triageContext
.ModelTimelineIssues
.AsNoTracking()
.Where(x => x.ModelBuildId == modelBuild.Id)
.CountAsync()
.ConfigureAwait(false);
if (count < 100)
if (totalCount < max)
{
return;
}

var timelineIssues = await triageContext
.ModelTimelineIssues
.Where(x => x.ModelBuildId == modelBuild.Id)
.ToListAsync()
.ConfigureAwait(false);
Logger.LogInformation($"Deleting {timelineIssues.Count} timeline issues");
triageContext.RemoveRange(timelineIssues);
await triageContext.SaveChangesAsync().ConfigureAwait(false);
Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many timeline issues ({totalCount})");
var deleted = 0;
do
{
var timelineIssueIds = await triageContext
.ModelTimelineIssues
.AsNoTracking()
.Where(x => x.ModelBuildId == modelBuild.Id)
.Take(max)
.Select(x => x.Id)
.ToListAsync()
.ConfigureAwait(false);
if (timelineIssueIds.Count == 0)
{
break;
}

triageContext.RemoveRange(timelineIssueIds.Select(x => new ModelTimelineIssue() { Id = x }));
deleted += timelineIssueIds.Count;
Logger.LogInformation($"Deleted {deleted} timeline issues of {totalCount}");
await triageContext.SaveChangesAsync().ConfigureAwait(false);
} while (true);
}

async Task DeleteTrackingIssueMatches(ModelBuild modelBuild)
Expand All @@ -128,15 +145,16 @@ async Task DeleteTrackingIssueMatches(ModelBuild modelBuild)
var count = 0;
foreach (var buildAttemptId in buildAttemptIds)
{
var trackingIssues = await triageContext
var trackingIssueIds = await triageContext
.ModelTrackingIssueMatches
.Where(x => x.ModelBuildAttemptId == buildAttemptId)
.Select(x => x.Id)
.ToListAsync()
.ConfigureAwait(false);
if (trackingIssues.Count > 0)
if (trackingIssueIds.Count > 0)
{
count += trackingIssues.Count;
triageContext.RemoveRange(trackingIssues);
count += trackingIssueIds.Count;
triageContext.RemoveRange(trackingIssueIds.Select(x => new ModelTrackingIssueMatch() { Id = x }));
}
}

Expand All @@ -161,25 +179,27 @@ async Task DeleteTestResults(ModelBuild modelBuild)
return;
}

Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many test resultts ({testCount})");
Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many test results ({testCount})");

var total = 0;

do
{
var testResults = await triageContext
var testResultIds = await triageContext
.ModelTestResults
.AsNoTracking()
.Where(x => x.ModelBuildId == modelBuild.Id)
.Take(100)
.Take(testCountMax)
.Select(x => x.Id)
.ToListAsync()
.ConfigureAwait(false);
if (testResults.Count == 0)
if (testResultIds.Count == 0)
{
break;
}

triageContext.RemoveRange(testResults);
total += testResults.Count;
triageContext.RemoveRange(testResultIds.Select(x => new ModelTestResult() { Id = x }));
total += testResultIds.Count;
Logger.LogInformation($"Deleted {total} test resuts out of {testCount} from {modelBuild.GetBuildKey()}");

await triageContext.SaveChangesAsync().ConfigureAwait(false);
Expand Down
2 changes: 1 addition & 1 deletion DevOps.Util.DotNet/Triage/SearchTestsRequest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public IQueryable<ModelTestResult> Filter(IQueryable<ModelTestResult> query)

if (!string.IsNullOrEmpty(WorkItemName))
{
query = query.Where(x => EF.Functions.Like(x.HelixWorkItemName, $"%{WorkItemName}%"));
query = query.Where(x => EF.Functions.Like(x.HelixWorkItemName!, $"%{WorkItemName}%"));
}

// Keep this in sync with logic in SearchTimelineRequest
Expand Down
5 changes: 5 additions & 0 deletions DevOps.Util.DotNet/Triage/StatusPageUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ public async Task<string> GetStatusIssueTextAsync(IGitHubClient gitHubClient)
header.AppendLine("Please use these queries to discover issues");

var runtimeDefinition = await TriageContextUtil.GetModelBuildDefinitionAsync(id: 686).ConfigureAwait(false);
if (runtimeDefinition is null)
{
return "Could not find definition";
}

var runtimeDefinitionKey = runtimeDefinition.GetDefinitionKey();

await BuildOne("Blocking CI", "blocking-clean-ci", runtimeDefinitionKey);
Expand Down
22 changes: 19 additions & 3 deletions DevOps.Util.DotNet/Triage/TriageContextUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ public IQueryable<ModelBuildDefinition> GetModelBuildDefinitionQueryAsync(int id
public Task<ModelBuildDefinition?> FindModelBuildDefinitionAsync(int id) =>
GetModelBuildDefinitionQueryAsync(id).FirstOrDefaultAsync()!;

public Task<ModelBuildDefinition> GetModelBuildDefinitionAsync(int id) =>
public Task<ModelBuildDefinition?> GetModelBuildDefinitionAsync(int id) =>
GetModelBuildDefinitionQueryAsync(id).SingleOrDefaultAsync();

public async Task<ModelBuildDefinition?> FindModelBuildDefinitionAsync(string azureOrganization, string nameOrId)
Expand Down Expand Up @@ -405,11 +405,11 @@ public async Task<ModelTestRun> EnsureTestRunAsync(ModelBuildAttempt modelBuildA
var testCaseResult = dotnetTestCaseResult.TestCaseResult;
var testResult = new ModelTestResult()
{
TestFullName = testCaseResult.TestCaseTitle,
TestFullName = NormalizeString(testCaseResult.TestCaseTitle),
Outcome = testCaseResult.Outcome,
ModelTestRun = modelTestRun,
TestRunName = modelTestRun.Name,
ErrorMessage = testCaseResult.ErrorMessage ?? "",
ErrorMessage = NormalizeString(testCaseResult.ErrorMessage),
IsSubResultContainer = testCaseResult.SubResults?.Length > 0,
IsSubResult = false,
StartTime = modelBuildAttempt.StartTime,
Expand Down Expand Up @@ -469,6 +469,22 @@ void AddQueryData(ModelTestResult testResult)
}
}

string NormalizeString(string? value)
{
if (value is null)
{
return "";
}

var maxLen = 4096;
if (value.Length > maxLen)
{
value = $"{value.AsSpan().Slice(0, maxLen)}...";
}

return value;
}

await Context.SaveChangesAsync().ConfigureAwait(false);
return modelTestRun;
}
Expand Down
5 changes: 3 additions & 2 deletions scratch/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,9 @@ internal static IConfiguration CreateConfiguration(bool useProduction = false)
internal async Task Scratch()
{
Reset(useProduction: true);
var util = new ModelDataUtil(DotNetQueryUtil, HelixServer, TriageContextUtil, CreateLogger());
await util.EnsureModelInfoAsync("public", 35950, includeTests: true);
await DeleteOldBuilds(useProduction: true);
// var util = new ModelDataUtil(DotNetQueryUtil, HelixServer, TriageContextUtil, CreateLogger());
// await util.EnsureModelInfoAsync("public", 35950, includeTests: true);
}

/// <summary>
Expand Down

0 comments on commit eeb9405

Please sign in to comment.