From e2ace39f122a674ef30dc954e5536a1b3f7becbb Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Jul 2021 17:48:30 +0200 Subject: [PATCH 1/2] Properly rebuild optimization data when it changes The timestamp of the merged .mibc file was set to when the tool was invoked, while the inputs have a timestamp from when they were created in the training scenarios. That means the target to create the merged .mibc file would not run incrementally until many weeks later. To fix add an --inherit-timestamp flag to dotnet-pgo that makes the merged output inherit the max timestamp from the inputs. This is not ideal as it means incrementally going backwards does not work, but it's better than the previous behavior. Fix #53637 --- src/coreclr/crossgen-corelib.proj | 1 + src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs | 3 +++ src/coreclr/tools/dotnet-pgo/Program.cs | 12 +++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj index 6d675a2a59e24..655fc3281eb2e 100644 --- a/src/coreclr/crossgen-corelib.proj +++ b/src/coreclr/crossgen-corelib.proj @@ -66,6 +66,7 @@ $(DotNetCli) $([MSBuild]::NormalizePath('$(BinDir)', 'dotnet-pgo', 'dotnet-pgo.dll')) merge $(DotNetPgoCmd) -o:$(MergedMibcPath) $(DotNetPgoCmd) @(OptimizationMibcFiles->'-i:%(Identity)', ' ') + $(DotNetPgoCmd) --inherit-timestamp diff --git a/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs b/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs index 3da9ded70632f..a2e3a82f37fdb 100644 --- a/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs +++ b/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs @@ -45,6 +45,7 @@ internal class CommandLineOptions public bool DumpMibc = false; public FileInfo InputFileToDump; public List CompareMibc; + public bool InheritTimestamp; public string[] HelpArgs = Array.Empty(); @@ -265,6 +266,8 @@ void HelpOption() } } + syntax.DefineOption(name: "inherit-timestamp", value: ref InheritTimestamp, help: "If specified, set the output's timestamp to the max timestamp of the input files"); + VerbosityOption(); CompressedOption(); HelpOption(); diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index f129fdfee59bf..95aa2c8a81a05 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -378,7 +378,17 @@ static int InnerMergeMain(CommandLineOptions commandLineOptions) ProfileData.MergeProfileData(ref partialNgen, mergedProfileData, MIbcProfileParser.ParseMIbcFile(tsc, peReader, assemblyNamesInBubble, onlyDefinedInAssembly: null)); } - return MibcEmitter.GenerateMibcFile(tsc, commandLineOptions.OutputFileName, mergedProfileData.Values, commandLineOptions.ValidateOutputFile, commandLineOptions.Uncompressed); + int result = MibcEmitter.GenerateMibcFile(tsc, commandLineOptions.OutputFileName, mergedProfileData.Values, commandLineOptions.ValidateOutputFile, commandLineOptions.Uncompressed); + if (result == 0 && commandLineOptions.InheritTimestamp) + { + static DateTime Latest(IEnumerable datetimes) + => datetimes.Aggregate((cur, dt) => dt > cur ? dt : cur); + + commandLineOptions.OutputFileName.CreationTimeUtc = Latest(commandLineOptions.InputFilesToMerge.Select(fi => fi.CreationTimeUtc)); + commandLineOptions.OutputFileName.LastWriteTimeUtc = Latest(commandLineOptions.InputFilesToMerge.Select(fi => fi.LastWriteTimeUtc)); + } + + return result; } finally { From 51b574f17f0000edd4f9a2f85bfca74268d6ff31 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Jul 2021 19:42:37 +0200 Subject: [PATCH 2/2] Use Enumerable.Max --- src/coreclr/tools/dotnet-pgo/Program.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index 95aa2c8a81a05..77113377c478d 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -381,11 +381,8 @@ static int InnerMergeMain(CommandLineOptions commandLineOptions) int result = MibcEmitter.GenerateMibcFile(tsc, commandLineOptions.OutputFileName, mergedProfileData.Values, commandLineOptions.ValidateOutputFile, commandLineOptions.Uncompressed); if (result == 0 && commandLineOptions.InheritTimestamp) { - static DateTime Latest(IEnumerable datetimes) - => datetimes.Aggregate((cur, dt) => dt > cur ? dt : cur); - - commandLineOptions.OutputFileName.CreationTimeUtc = Latest(commandLineOptions.InputFilesToMerge.Select(fi => fi.CreationTimeUtc)); - commandLineOptions.OutputFileName.LastWriteTimeUtc = Latest(commandLineOptions.InputFilesToMerge.Select(fi => fi.LastWriteTimeUtc)); + commandLineOptions.OutputFileName.CreationTimeUtc = commandLineOptions.InputFilesToMerge.Max(fi => fi.CreationTimeUtc); + commandLineOptions.OutputFileName.LastWriteTimeUtc = commandLineOptions.InputFilesToMerge.Max(fi => fi.LastWriteTimeUtc); } return result;