From f4098a18cbb62c78453b52b944d6b7516a3d5e50 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 17 Mar 2021 15:26:37 -0700 Subject: [PATCH 1/7] Build changes to consume mibc profile data --- Directory.Build.props | 1 + eng/Versions.props | 5 ++- eng/restore/optimizationData.targets | 37 +++++++++++++++++++ src/coreclr/crossgen-corelib.proj | 6 +++ .../Microsoft.NETCore.App/ReadyToRun.targets | 14 ++++++- 5 files changed, 61 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index f971fa0a774fcc..915597f44963cc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -60,6 +60,7 @@ $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', 'tools-local')) $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', 'src', 'tasks')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'ibc')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'mibc')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'docs')) $([MSBuild]::NormalizeDirectory('$(NuGetPackageRoot)', 'microsoft.private.intellisense', '$(MicrosoftPrivateIntellisenseVersion)', 'IntellisenseFiles', 'net')) $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', 'docs')) diff --git a/eng/Versions.props b/eng/Versions.props index 4c9cddddeec995..bb91951933ccba 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -17,7 +17,7 @@ true true - true + false false + 99.99.99-master-20210317.2 + 99.99.99-master-20210317.2 + 99.99.99-master-20210317.2 99.99.99-master-20200806.6 99.99.99-master-20200806.6 99.99.99-master-20200806.6 diff --git a/eng/restore/optimizationData.targets b/eng/restore/optimizationData.targets index 9b431aa953af2d..7110cdea2b0abf 100644 --- a/eng/restore/optimizationData.targets +++ b/eng/restore/optimizationData.targets @@ -34,4 +34,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + $(NuGetPackageRoot)%(MIBCPackage.Identity)\%(MIBCPackage.Version) + + <_optimizationMibcFile Include="%(MIBCPackage.PackagePath)\**\*.mibc" SubdirectoryName="$(TargetOS)/$(TargetArchitecture)" /> + + + + + + + diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj index b2a1848ed51f36..3915184e1a2d9d 100644 --- a/src/coreclr/crossgen-corelib.proj +++ b/src/coreclr/crossgen-corelib.proj @@ -70,11 +70,17 @@ + + + + $(DotNetCli) $([MSBuild]::NormalizePath('$(BinDir)', '$(CrossDir)', 'crossgen2', 'crossgen2.dll')) $(CrossGenDllCmd) -o:$(CoreLibOutputPath) $(CrossGenDllCmd) -r:$([MSBuild]::NormalizePath('$(BinDir)', 'IL', '*.dll')) $(CrossGenDllCmd) --targetarch:$(TargetArchitecture) + @(OptimizationMibcFiles->'-m:%(Identity)', ' ') + $(CrossGenDllCmd) $(MibcArgs) --embed-pgo-data $(CrossGenDllCmd) -O $(CrossGenDllCmd) $(CoreLibInputPath) diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets index b898cdaea116ae..aaf98074c871bf 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets @@ -1,5 +1,17 @@ + + + + + + + + + + + + @@ -7,7 +19,7 @@ $(CoreCLRArtifactsPath)\$(CrossDir)\crossgen2 $(Crossgen2Dir)\crossgen2$(ExeSuffix) true - --targetarch:$(TargetArchitecture) + @(PublishReadyToRunCrossgen2ExtraArgsList) unix win From 6ff02cb58f95562f2b2004dbddc3b7a982e68274 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 17 Mar 2021 15:52:36 -0700 Subject: [PATCH 2/7] Handle incremental build correctly --- eng/restore/optimizationData.targets | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eng/restore/optimizationData.targets b/eng/restore/optimizationData.targets index 7110cdea2b0abf..3d97d5cc8a8642 100644 --- a/eng/restore/optimizationData.targets +++ b/eng/restore/optimizationData.targets @@ -49,6 +49,9 @@ + + + @@ -61,10 +64,16 @@ $(NuGetPackageRoot)%(MIBCPackage.Identity)\%(MIBCPackage.Version) <_optimizationMibcFile Include="%(MIBCPackage.PackagePath)\**\*.mibc" SubdirectoryName="$(TargetOS)/$(TargetArchitecture)" /> + + + + + Date: Wed, 17 Mar 2021 17:47:00 -0700 Subject: [PATCH 3/7] Add dump feature to dotnet-pgo --- .../tools/dotnet-pgo/CommandLineOptions.cs | 24 +++- src/coreclr/tools/dotnet-pgo/Program.cs | 104 ++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs b/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs index ecf68576a8403c..70a8b2d081d904 100644 --- a/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs +++ b/src/coreclr/tools/dotnet-pgo/CommandLineOptions.cs @@ -38,6 +38,8 @@ internal class CommandLineOptions public bool DetailedProgressMessages; public List InputFilesToMerge; public List IncludedAssemblies = new List(); + public bool DumpMibc = false; + public FileInfo InputFileToDump; public string[] HelpArgs = Array.Empty(); @@ -259,6 +261,26 @@ void HelpOption() #endif } + var dumpCommand = syntax.DefineCommand(name: "dump", value: ref command, help: "Dump the contents of a Mibc file."); + if (dumpCommand.IsActive) + { + DumpMibc = true; + HelpArgs = new string[] { "dump", "--help", "input", "output" }; + + VerbosityOption(); + HelpOption(); + + string inputFileToDump = null; + syntax.DefineParameter(name: "input", ref inputFileToDump, "Name of the input mibc file to dump."); + if (inputFileToDump != null) + InputFileToDump = new FileInfo(inputFileToDump); + + string outputFile = null; + syntax.DefineParameter(name: "output", ref outputFile, "Name of the output dump file."); + if (outputFile != null) + OutputFileName = new FileInfo(outputFile); + } + if (syntax.ActiveCommand == null) { @@ -341,7 +363,7 @@ private CommandLineOptions() private void ParseCommmandLineHelper(string[] args) { ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, DefineArgumentSyntax); - if (Help || (!FileType.HasValue && (InputFilesToMerge == null))) + if (Help || (!FileType.HasValue && (InputFilesToMerge == null) && !DumpMibc)) { Help = true; } diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index e097a9a1c312ed..86448cfb57cbe7 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -28,6 +28,9 @@ using System.Reflection.PortableExecutable; using ILCompiler.IBC; using ILCompiler; +using System.Runtime.Serialization.Json; +using System.Text.Json; +using System.Text.Encodings.Web; namespace Microsoft.Diagnostics.Tools.Pgo { @@ -241,6 +244,10 @@ static int InnerMain(CommandLineOptions commandLineOptions) if (!commandLineOptions.DetailedProgressMessages) s_logger.HideDetailedMessages(); + if (commandLineOptions.DumpMibc) + { + return InnerDumpMain(commandLineOptions); + } if (commandLineOptions.InputFilesToMerge != null) { return InnerMergeMain(commandLineOptions); @@ -251,6 +258,103 @@ static int InnerMain(CommandLineOptions commandLineOptions) } } + static int InnerDumpMain(CommandLineOptions commandLineOptions) + { + if ((commandLineOptions.InputFileToDump == null) || (!commandLineOptions.InputFileToDump.Exists)) + { + PrintUsage(commandLineOptions, "Valid input file must be specified"); + return -8; + } + + if (commandLineOptions.OutputFileName == null) + { + PrintUsage(commandLineOptions, "Output filename must be specified"); + return -8; + } + + PrintDetailedMessage($"Opening {commandLineOptions.InputFileToDump}"); + var mibcPeReader = MIbcProfileParser.OpenMibcAsPEReader(commandLineOptions.InputFileToDump.FullName); + var tsc = new TypeRefTypeSystem.TypeRefTypeSystemContext(new PEReader[] { mibcPeReader }); + + PrintDetailedMessage($"Parsing {commandLineOptions.InputFileToDump}"); + var profileData = MIbcProfileParser.ParseMIbcFile(tsc, mibcPeReader, null, onlyDefinedInAssembly: null); + + using (FileStream outputFile = new FileStream(commandLineOptions.OutputFileName.FullName, FileMode.Create, FileAccess.Write)) + { + JsonWriterOptions options = new JsonWriterOptions(); + options.Indented = true; + options.SkipValidation = false; + options.Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping; + + using Utf8JsonWriter jsonWriter = new Utf8JsonWriter(outputFile, options); + jsonWriter.WriteStartObject(); + jsonWriter.WriteStartArray("Methods"); + foreach (MethodProfileData data in profileData.GetAllMethodProfileData()) + { + jsonWriter.WriteStartObject(); + jsonWriter.WriteString("Method", data.Method.ToString()); + if (data.CallWeights != null) + { + jsonWriter.WriteStartArray("CallWeights"); + foreach (var callWeight in data.CallWeights) + { + jsonWriter.WriteString("Method", callWeight.Key.ToString()); + jsonWriter.WriteNumber("Weight", callWeight.Value); + } + jsonWriter.WriteEndArray(); + } + if (data.ExclusiveWeight != 0) + { + jsonWriter.WriteNumber("ExclusiveWeight", data.ExclusiveWeight); + } + if (data.SchemaData != null) + { + jsonWriter.WriteStartArray("InstrumentationData"); + foreach (var schemaElem in data.SchemaData) + { + jsonWriter.WriteStartObject(); + jsonWriter.WriteNumber("ILOffset", schemaElem.ILOffset); + jsonWriter.WriteString("InstrumentationKind", schemaElem.InstrumentationKind.ToString()); + jsonWriter.WriteNumber("Other", schemaElem.Other); + if (schemaElem.DataHeldInDataLong) + { + jsonWriter.WriteNumber("Data", schemaElem.DataLong); + } + else + { + if (schemaElem.DataObject == null) + { + // No data associated with this item + } + else if (schemaElem.DataObject.Length == 1) + { + jsonWriter.WriteString("Data", schemaElem.DataObject.GetValue(0).ToString()); + } + else + { + jsonWriter.WriteStartArray("Data"); + foreach (var dataElem in schemaElem.DataObject) + { + jsonWriter.WriteStringValue(dataElem.ToString()); + } + jsonWriter.WriteEndArray(); + } + } + jsonWriter.WriteEndObject(); + } + jsonWriter.WriteEndArray(); + } + + jsonWriter.WriteEndObject(); + } + jsonWriter.WriteEndArray(); + jsonWriter.WriteEndObject(); + } + PrintMessage($"Generated {commandLineOptions.OutputFileName}"); + + return 0; + } + static int InnerMergeMain(CommandLineOptions commandLineOptions) { From 21bea4664ac9708b6593eb9ac4a9db265a77e962 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 17 Mar 2021 19:22:42 -0700 Subject: [PATCH 4/7] Apply static pgo data to release builds only --- src/coreclr/crossgen-corelib.proj | 2 +- src/coreclr/jit/fgprofile.cpp | 5 ++++- .../pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj index 3915184e1a2d9d..13e93155ef87a5 100644 --- a/src/coreclr/crossgen-corelib.proj +++ b/src/coreclr/crossgen-corelib.proj @@ -80,7 +80,7 @@ $(CrossGenDllCmd) -r:$([MSBuild]::NormalizePath('$(BinDir)', 'IL', '*.dll')) $(CrossGenDllCmd) --targetarch:$(TargetArchitecture) @(OptimizationMibcFiles->'-m:%(Identity)', ' ') - $(CrossGenDllCmd) $(MibcArgs) --embed-pgo-data + $(CrossGenDllCmd) $(MibcArgs) --embed-pgo-data $(CrossGenDllCmd) -O $(CrossGenDllCmd) $(CoreLibInputPath) diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index f32137f1404e68..6c24aea073f5a2 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -2445,7 +2445,10 @@ void EfficientEdgeCountReconstructor::Propagate() { // We don't expect mismatches or convergence failures. // - assert(!m_mismatch); + + // Mismatches are currently expected as the flow for static pgo doesn't prevent them now. + // assert(!m_mismatch); + assert(!m_failedToConverge); // If any issues arose during reconstruction, don't set weights. diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets index aaf98074c871bf..336b5c81c78cfe 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets @@ -8,8 +8,8 @@ - - + + From 3cb22734cd53a96d641f65690fbcd57e36a7351d Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 18 Mar 2021 13:35:53 -0700 Subject: [PATCH 5/7] Fix Unix path character and case sensitivity issues --- eng/restore/optimizationData.targets | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/restore/optimizationData.targets b/eng/restore/optimizationData.targets index 3d97d5cc8a8642..8e711d9cbe6539 100644 --- a/eng/restore/optimizationData.targets +++ b/eng/restore/optimizationData.targets @@ -37,20 +37,20 @@ - - - + + + - - - + + + - + @@ -61,11 +61,11 @@ - $(NuGetPackageRoot)%(MIBCPackage.Identity)\%(MIBCPackage.Version) + $(NuGetPackageRoot)%(MIBCPackage.Identity)/%(MIBCPackage.Version) - <_optimizationMibcFile Include="%(MIBCPackage.PackagePath)\**\*.mibc" SubdirectoryName="$(TargetOS)/$(TargetArchitecture)" /> + <_optimizationMibcFile Include="%(MIBCPackage.PackagePath)/**/*.mibc" SubdirectoryName="$(TargetOS)/$(TargetArchitecture)" /> + Exclude="@(_optimizationMibcFile->'$(MibcOptimizationDataDir)%(SubdirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')"/> @@ -75,7 +75,7 @@ From e0a811314057c5730f38f213e312256c94d7ef7b Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 18 Mar 2021 15:02:31 -0700 Subject: [PATCH 6/7] code review feedback --- eng/restore/optimizationData.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/restore/optimizationData.targets b/eng/restore/optimizationData.targets index 8e711d9cbe6539..c7ce30fadd249e 100644 --- a/eng/restore/optimizationData.targets +++ b/eng/restore/optimizationData.targets @@ -37,7 +37,7 @@ - + From 642c85bab6570b02467d4e3a628ba32c06e02ad8 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 18 Mar 2021 15:05:08 -0700 Subject: [PATCH 7/7] Fix formatting --- src/coreclr/jit/fgprofile.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 6c24aea073f5a2..8c158ddc458a19 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -2445,10 +2445,10 @@ void EfficientEdgeCountReconstructor::Propagate() { // We don't expect mismatches or convergence failures. // - + // Mismatches are currently expected as the flow for static pgo doesn't prevent them now. - // assert(!m_mismatch); - + // assert(!m_mismatch); + assert(!m_failedToConverge); // If any issues arose during reconstruction, don't set weights.