diff --git a/.vscode/launch.json b/.vscode/launch.json index 55e6d63..a27fb33 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,7 +26,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/scratch/bin/Debug/netcoreapp3.1/scratch.dll", + "program": "${workspaceFolder}/scratch/bin/Debug/net6.0/scratch.dll", "args": [ ], "cwd": "${workspaceFolder}/scratch", diff --git a/DevOps.Util.DotNet/Function/FunctionUtil.cs b/DevOps.Util.DotNet/Function/FunctionUtil.cs index b6a0f75..79ffca1 100644 --- a/DevOps.Util.DotNet/Function/FunctionUtil.cs +++ b/DevOps.Util.DotNet/Function/FunctionUtil.cs @@ -52,12 +52,13 @@ public async Task OnPullRequestMergedAsync( /// public async Task DeleteOldBuilds(TriageContext triageContext, int deleteMax = 25) { - var limitDays = 30; + var limitDays = 21; var limit = DateTime.UtcNow - TimeSpan.FromDays(limitDays); var count = 0; var builds = await triageContext .ModelBuilds + .AsNoTracking() .Where(x => x.StartTime < limit) .OrderBy(x => x.StartTime) .Take(deleteMax) @@ -96,24 +97,40 @@ public async Task DeleteOldBuilds(TriageContext triageContext, int deleteMa async Task DeleteTimelineIssues(ModelBuild modelBuild) { - var count = await triageContext + var max = 100; + var totalCount = await triageContext .ModelTimelineIssues + .AsNoTracking() .Where(x => x.ModelBuildId == modelBuild.Id) .CountAsync() .ConfigureAwait(false); - if (count < 100) + if (totalCount < max) { return; } - var timelineIssues = await triageContext - .ModelTimelineIssues - .Where(x => x.ModelBuildId == modelBuild.Id) - .ToListAsync() - .ConfigureAwait(false); - Logger.LogInformation($"Deleting {timelineIssues.Count} timeline issues"); - triageContext.RemoveRange(timelineIssues); - await triageContext.SaveChangesAsync().ConfigureAwait(false); + Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many timeline issues ({totalCount})"); + var deleted = 0; + do + { + var timelineIssueIds = await triageContext + .ModelTimelineIssues + .AsNoTracking() + .Where(x => x.ModelBuildId == modelBuild.Id) + .Take(max) + .Select(x => x.Id) + .ToListAsync() + .ConfigureAwait(false); + if (timelineIssueIds.Count == 0) + { + break; + } + + triageContext.RemoveRange(timelineIssueIds.Select(x => new ModelTimelineIssue() { Id = x })); + deleted += timelineIssueIds.Count; + Logger.LogInformation($"Deleted {deleted} timeline issues of {totalCount}"); + await triageContext.SaveChangesAsync().ConfigureAwait(false); + } while (true); } async Task DeleteTrackingIssueMatches(ModelBuild modelBuild) @@ -128,15 +145,16 @@ async Task DeleteTrackingIssueMatches(ModelBuild modelBuild) var count = 0; foreach (var buildAttemptId in buildAttemptIds) { - var trackingIssues = await triageContext + var trackingIssueIds = await triageContext .ModelTrackingIssueMatches .Where(x => x.ModelBuildAttemptId == buildAttemptId) + .Select(x => x.Id) .ToListAsync() .ConfigureAwait(false); - if (trackingIssues.Count > 0) + if (trackingIssueIds.Count > 0) { - count += trackingIssues.Count; - triageContext.RemoveRange(trackingIssues); + count += trackingIssueIds.Count; + triageContext.RemoveRange(trackingIssueIds.Select(x => new ModelTrackingIssueMatch() { Id = x })); } } @@ -161,25 +179,27 @@ async Task DeleteTestResults(ModelBuild modelBuild) return; } - Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many test resultts ({testCount})"); + Logger.LogInformation($"{modelBuild.GetBuildKey()} has too many test results ({testCount})"); var total = 0; do { - var testResults = await triageContext + var testResultIds = await triageContext .ModelTestResults + .AsNoTracking() .Where(x => x.ModelBuildId == modelBuild.Id) - .Take(100) + .Take(testCountMax) + .Select(x => x.Id) .ToListAsync() .ConfigureAwait(false); - if (testResults.Count == 0) + if (testResultIds.Count == 0) { break; } - triageContext.RemoveRange(testResults); - total += testResults.Count; + triageContext.RemoveRange(testResultIds.Select(x => new ModelTestResult() { Id = x })); + total += testResultIds.Count; Logger.LogInformation($"Deleted {total} test resuts out of {testCount} from {modelBuild.GetBuildKey()}"); await triageContext.SaveChangesAsync().ConfigureAwait(false); diff --git a/DevOps.Util.DotNet/Triage/SearchTestsRequest.cs b/DevOps.Util.DotNet/Triage/SearchTestsRequest.cs index 458610f..112cc43 100644 --- a/DevOps.Util.DotNet/Triage/SearchTestsRequest.cs +++ b/DevOps.Util.DotNet/Triage/SearchTestsRequest.cs @@ -46,7 +46,7 @@ public IQueryable Filter(IQueryable query) if (!string.IsNullOrEmpty(WorkItemName)) { - query = query.Where(x => EF.Functions.Like(x.HelixWorkItemName, $"%{WorkItemName}%")); + query = query.Where(x => EF.Functions.Like(x.HelixWorkItemName!, $"%{WorkItemName}%")); } // Keep this in sync with logic in SearchTimelineRequest diff --git a/DevOps.Util.DotNet/Triage/StatusPageUtil.cs b/DevOps.Util.DotNet/Triage/StatusPageUtil.cs index 7daa33a..63d5770 100644 --- a/DevOps.Util.DotNet/Triage/StatusPageUtil.cs +++ b/DevOps.Util.DotNet/Triage/StatusPageUtil.cs @@ -68,6 +68,11 @@ public async Task GetStatusIssueTextAsync(IGitHubClient gitHubClient) header.AppendLine("Please use these queries to discover issues"); var runtimeDefinition = await TriageContextUtil.GetModelBuildDefinitionAsync(id: 686).ConfigureAwait(false); + if (runtimeDefinition is null) + { + return "Could not find definition"; + } + var runtimeDefinitionKey = runtimeDefinition.GetDefinitionKey(); await BuildOne("Blocking CI", "blocking-clean-ci", runtimeDefinitionKey); diff --git a/DevOps.Util.DotNet/Triage/TriageContextUtil.cs b/DevOps.Util.DotNet/Triage/TriageContextUtil.cs index dfa6abc..b2f7fb6 100644 --- a/DevOps.Util.DotNet/Triage/TriageContextUtil.cs +++ b/DevOps.Util.DotNet/Triage/TriageContextUtil.cs @@ -354,7 +354,7 @@ public IQueryable GetModelBuildDefinitionQueryAsync(int id public Task FindModelBuildDefinitionAsync(int id) => GetModelBuildDefinitionQueryAsync(id).FirstOrDefaultAsync()!; - public Task GetModelBuildDefinitionAsync(int id) => + public Task GetModelBuildDefinitionAsync(int id) => GetModelBuildDefinitionQueryAsync(id).SingleOrDefaultAsync(); public async Task FindModelBuildDefinitionAsync(string azureOrganization, string nameOrId) @@ -405,11 +405,11 @@ public async Task EnsureTestRunAsync(ModelBuildAttempt modelBuildA var testCaseResult = dotnetTestCaseResult.TestCaseResult; var testResult = new ModelTestResult() { - TestFullName = testCaseResult.TestCaseTitle, + TestFullName = NormalizeString(testCaseResult.TestCaseTitle), Outcome = testCaseResult.Outcome, ModelTestRun = modelTestRun, TestRunName = modelTestRun.Name, - ErrorMessage = testCaseResult.ErrorMessage ?? "", + ErrorMessage = NormalizeString(testCaseResult.ErrorMessage), IsSubResultContainer = testCaseResult.SubResults?.Length > 0, IsSubResult = false, StartTime = modelBuildAttempt.StartTime, @@ -469,6 +469,22 @@ void AddQueryData(ModelTestResult testResult) } } + string NormalizeString(string? value) + { + if (value is null) + { + return ""; + } + + var maxLen = 4096; + if (value.Length > maxLen) + { + value = $"{value.AsSpan().Slice(0, maxLen)}..."; + } + + return value; + } + await Context.SaveChangesAsync().ConfigureAwait(false); return modelTestRun; } diff --git a/scratch/Program.cs b/scratch/Program.cs index 4092870..29dd7e2 100644 --- a/scratch/Program.cs +++ b/scratch/Program.cs @@ -173,8 +173,9 @@ internal static IConfiguration CreateConfiguration(bool useProduction = false) internal async Task Scratch() { Reset(useProduction: true); - var util = new ModelDataUtil(DotNetQueryUtil, HelixServer, TriageContextUtil, CreateLogger()); - await util.EnsureModelInfoAsync("public", 35950, includeTests: true); + await DeleteOldBuilds(useProduction: true); + // var util = new ModelDataUtil(DotNetQueryUtil, HelixServer, TriageContextUtil, CreateLogger()); + // await util.EnsureModelInfoAsync("public", 35950, includeTests: true); } ///