From b8c961045a9272197197d8acc4a53b159656310e Mon Sep 17 00:00:00 2001 From: Osvaldo Calles Date: Wed, 7 Sep 2022 17:42:56 -0700 Subject: [PATCH 1/2] Add new output DebugSymbolsFiles to ResolvePackageAssets DebugSymbolsFiles outputs the list of absolute path to symbols files for package references. ReferenceDocumentationFiles outputs the list of absolute path to Xml documentation files. The change: ResolvePackagesAssets was modified to read/write DebugSymbolsFiles and ReferenceDocumentationFiles from/to the cache file. They were inserted after ContentFilesToProcess to keep the alphabetical order. The target was modified to copy the DebugSymbolsFiles to output directory only when CopyDebugSymbolFilesFromPackages is true, and ReferenceDocumentationFiles to the output directory when EnableCopySymbolsAndXmlDocsToOutputDir is true. There is some minor code cleanup not related to this bug. --- src/Tasks/Common/MetadataKeys.cs | 7 ++ .../ResolvePackageAssets.cs | 89 ++++++++++++++++++- ...rosoft.PackageDependencyResolution.targets | 10 +++ 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/src/Tasks/Common/MetadataKeys.cs b/src/Tasks/Common/MetadataKeys.cs index 781236d51932..0facb5d87b4f 100644 --- a/src/Tasks/Common/MetadataKeys.cs +++ b/src/Tasks/Common/MetadataKeys.cs @@ -131,5 +131,12 @@ internal static class MetadataKeys public const string IsVersion5 = "IsVersion5"; public const string CreateCompositeImage = "CreateCompositeImage"; public const string PerfmapFormatVersion = "PerfmapFormatVersion"; + + // Debug symbols + public const string RelatedProperty = "related"; + public const string XmlExtension = ".xml"; + public const string XmlFilePath = "XmlFilePath"; + public const string PdbExtension = ".pdb"; + public const string PdbFilePath = "PdbFilePath"; } } diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/ResolvePackageAssets.cs b/src/Tasks/Microsoft.NET.Build.Tasks/ResolvePackageAssets.cs index a6c7836dfbe3..aa6a3257e2fe 100644 --- a/src/Tasks/Microsoft.NET.Build.Tasks/ResolvePackageAssets.cs +++ b/src/Tasks/Microsoft.NET.Build.Tasks/ResolvePackageAssets.cs @@ -12,7 +12,6 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using NuGet.Common; -using NuGet.Frameworks; using NuGet.ProjectModel; using NuGet.Versioning; @@ -156,6 +155,9 @@ public sealed class ResolvePackageAssets : TaskBase [Required] public string DotNetAppHostExecutableNameWithoutExtension { get; set; } + /// + /// True indicates we are doing a design-time build. Otherwise we are in a build. + /// public bool DesignTimeBuild { get; set; } /// @@ -231,6 +233,24 @@ public sealed class ResolvePackageAssets : TaskBase [Output] public ITaskItem[] PackageDependencies { get; private set; } + /// + /// List of symbol files (.pdb) related to NuGet packages. + /// + /// + /// Pdb files to be copied to the output directory + /// + [Output] + public ITaskItem[] DebugSymbolsFiles { get; private set;} + + /// + /// List of xml files related to NuGet packages. + /// + /// + /// The XML files should only be included in the publish output if PublishReferencesDocumentationFiles is true + /// + [Output] + public ITaskItem[] ReferenceDocumentationFiles { get; private set; } + /// /// Messages from the assets file. /// These are logged directly and therefore not returned to the targets (note private here). @@ -311,11 +331,13 @@ private void ReadItemGroups() ApphostsForShimRuntimeIdentifiers = reader.ReadItemGroup(); CompileTimeAssemblies = reader.ReadItemGroup(); ContentFilesToPreprocess = reader.ReadItemGroup(); + DebugSymbolsFiles = reader.ReadItemGroup(); FrameworkAssemblies = reader.ReadItemGroup(); FrameworkReferences = reader.ReadItemGroup(); NativeLibraries = reader.ReadItemGroup(); PackageDependencies = reader.ReadItemGroup(); PackageFolders = reader.ReadItemGroup(); + ReferenceDocumentationFiles = reader.ReadItemGroup(); ResourceAssemblies = reader.ReadItemGroup(); RuntimeAssemblies = reader.ReadItemGroup(); RuntimeTargets = reader.ReadItemGroup(); @@ -510,7 +532,7 @@ private static BinaryReader CreateReaderFromDisk(ResolvePackageAssets task, byte BinaryReader reader = null; try { - if (File.GetLastWriteTimeUtc(task.ProjectAssetsCacheFile) > File.GetLastWriteTimeUtc(task.ProjectAssetsFile)) + if (IsCacheFileUpToDate()) { reader = OpenCacheFile(task.ProjectAssetsCacheFile, settingsHash); } @@ -537,6 +559,8 @@ private static BinaryReader CreateReaderFromDisk(ResolvePackageAssets task, byte } return reader; + + bool IsCacheFileUpToDate() => File.GetLastWriteTimeUtc(task.ProjectAssetsCacheFile) > File.GetLastWriteTimeUtc(task.ProjectAssetsFile); } private static BinaryReader OpenCacheStream(Stream stream, byte[] settingsHash) @@ -651,6 +675,8 @@ internal sealed class CacheWriter : IDisposable private const string NetCorePlatformLibrary = "Microsoft.NETCore.App"; + private const char RelatedPropertySeparator = ';'; + public CacheWriter(ResolvePackageAssets task) { _targetFramework = task.TargetFramework; @@ -776,11 +802,13 @@ private void WriteItemGroups() WriteItemGroup(WriteApphostsForShimRuntimeIdentifiers); WriteItemGroup(WriteCompileTimeAssemblies); WriteItemGroup(WriteContentFilesToPreprocess); + WriteItemGroup(WriteDebugSymbolsFiles); WriteItemGroup(WriteFrameworkAssemblies); WriteItemGroup(WriteFrameworkReferences); WriteItemGroup(WriteNativeLibraries); WriteItemGroup(WritePackageDependencies); - WriteItemGroup(WritePackageFolders); + WriteItemGroup(WritePackageFolders); + WriteItemGroup(WriteReferenceDocumentationFiles); WriteItemGroup(WriteResourceAssemblies); WriteItemGroup(WriteRuntimeAssemblies); WriteItemGroup(WriteRuntimeTargets); @@ -1091,6 +1119,61 @@ private void WriteContentFilesToPreprocess() }); } + private void WriteDebugSymbolsFiles() + { + WriteDebugItems( + p => p.RuntimeAssemblies, + MetadataKeys.PdbExtension); + } + + private void WriteReferenceDocumentationFiles() + { + WriteDebugItems( + p => p.CompileTimeAssemblies, + MetadataKeys.XmlExtension); + } + + private void WriteDebugItems( + Func> getAssets, + string extension) + { + foreach (var library in _runtimeTarget.Libraries) + { + if (!library.IsPackage()) + { + continue; + } + + foreach (LockFileItem asset in getAssets(library)) + { + if (asset.IsPlaceholderFile() || !asset.Properties.ContainsKey(MetadataKeys.RelatedProperty)) + { + continue; + } + + string itemSpec = _packageResolver.ResolvePackageAssetPath(library, asset.Path); + + string relatedExtensions = asset.Properties[MetadataKeys.RelatedProperty]; + + foreach (string fileExtension in relatedExtensions.Split(RelatedPropertySeparator)) + { + if (fileExtension.ToLower() == extension) + { + string xmlFilePath = Path.ChangeExtension(itemSpec, fileExtension); + if (File.Exists(xmlFilePath)) + { + WriteItem(xmlFilePath, library); + } + else + { + _task.Log.LogWarning(Strings.AssetsFileNotFound, xmlFilePath); + } + } + } + } + } + } + private void WriteFrameworkAssemblies() { if (_task.DisableFrameworkAssemblies) diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.PackageDependencyResolution.targets b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.PackageDependencyResolution.targets index d7347134a6f5..2456989cbf34 100644 --- a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.PackageDependencyResolution.targets +++ b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.PackageDependencyResolution.targets @@ -297,6 +297,8 @@ Copyright (c) .NET Foundation. All rights reserved. + + @@ -309,6 +311,14 @@ Copyright (c) .NET Foundation. All rights reserved. + + + + + + + + <_NativeRestoredAppHostNETCore Include="@(NativeCopyLocalItems)" Condition="'%(NativeCopyLocalItems.FileName)%(NativeCopyLocalItems.Extension)' == '$(_DotNetAppHostExecutableName)'"/> From 0debac3cb8cec2afc252f8b7a953bc21feb88339 Mon Sep 17 00:00:00 2001 From: Osvaldo Calles Date: Wed, 7 Sep 2022 17:43:35 -0700 Subject: [PATCH 2/2] Add unit-tests for copy debug symbols and documentation to output directory - Testing CopyDebugSymbolFilesFromPackages and CopyDocumentationFilesFromPackagesAdd unit-tests for EnableCopySymbolsAndXmlDocsToOutputDir --- .../GivenThatWeWantToBuildADesktopExe.cs | 226 +++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/src/Tests/Microsoft.NET.Build.Tests/GivenThatWeWantToBuildADesktopExe.cs b/src/Tests/Microsoft.NET.Build.Tests/GivenThatWeWantToBuildADesktopExe.cs index 3e2f9987c8cc..8d8f74953694 100644 --- a/src/Tests/Microsoft.NET.Build.Tests/GivenThatWeWantToBuildADesktopExe.cs +++ b/src/Tests/Microsoft.NET.Build.Tests/GivenThatWeWantToBuildADesktopExe.cs @@ -19,6 +19,8 @@ using Xunit.Abstractions; using System.Text.RegularExpressions; using System.Collections.Generic; +using System.Security.Cryptography; +using System.Reflection.Metadata; namespace Microsoft.NET.Build.Tests { @@ -530,7 +532,7 @@ void Test_inbox_assembly_wins_conflict_resolution(bool useSdkProject, string htt testProject.PackageReferences.Add(new TestPackageReference("System.Net.Http", httpPackageVersion)); - testProject.SourceFiles["Program.cs"] = + testProject.SourceFiles["Program.cs"] = (useAlias ? "extern alias snh;" + Environment.NewLine : "") + @"using System; @@ -850,5 +852,227 @@ public void It_allows_TargetFrameworkVersion_to_be_capitalized() .Should() .Pass(); } + + [WindowsOnlyFact] + public void It_places_package_xml_in_ref_folder_in_output_directory() + { + var testProject = new TestProject() + { + Name = "DesktopUsingPackageWithdXmlInRefFolder", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = true, + }; + + // This package contains an xml in ref, but not in lib. + testProject.PackageReferences.Add(new TestPackageReference("system.diagnostics.debug", "4.3.0")); + + testProject.AdditionalProperties.Add("CopyDebugSymbolFilesFromPackages", "true"); + testProject.AdditionalProperties.Add("CopyDocumentationFilesFromPackages", "true"); + + TestAsset testAsset = _testAssetsManager.CreateTestProject(testProject, testProject.Name); + + var buildCommand = new BuildCommand(testAsset); + + buildCommand + .Execute() + .Should() + .Pass(); + + var outputDirectory = buildCommand.GetOutputDirectory(testProject.TargetFrameworks); + + // this xml is coming from ref folder + outputDirectory.Should().HaveFile("System.Diagnostics.Debug.xml"); + } + + [WindowsOnlyTheory] + [InlineData("true", "true")] + [InlineData("true", "false")] + [InlineData("false", "true")] + [InlineData("false", "false")] + public void It_places_package_pdb_and_xml_files_in_output_directory(string enableCopyDebugSymbolFilesFromPackages, string enableDocumentationFilesFromPackages) + { + var testProject = new TestProject() + { + Name = "DesktopUsingPackageWithPdbAndXml", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = true, + }; + + testProject.PackageReferences.Add(new TestPackageReference("Microsoft.Build", "17.3.1")); + + testProject.AdditionalProperties.Add("CopyDebugSymbolFilesFromPackages", enableCopyDebugSymbolFilesFromPackages); + testProject.AdditionalProperties.Add("CopyDocumentationFilesFromPackages", enableDocumentationFilesFromPackages); + + string testPath = enableCopyDebugSymbolFilesFromPackages + enableDocumentationFilesFromPackages; + TestAsset testAsset = _testAssetsManager.CreateTestProject(testProject, testProject.Name, identifier: testPath); + + var buildCommand = new BuildCommand(testAsset); + + buildCommand + .Execute() + .Should() + .Pass(); + + var outputDirectory = buildCommand.GetOutputDirectory(testProject.TargetFrameworks); + + HelperCheckPdbAndDocumentation(outputDirectory, "Microsoft.Build", enableCopyDebugSymbolFilesFromPackages, enableDocumentationFilesFromPackages); + } + + [WindowsOnlyTheory] + [InlineData("true", "true")] + [InlineData("true", "false")] + [InlineData("false", "true")] + [InlineData("false", "false")] + public void It_places_package_pdb_and_xml_files_from_project_references_in_output_directory(string enableCopyDebugSymbolFilesFromPackages, string enableDocumentationFilesFromPackages) + { + var libraryProject = new TestProject() + { + Name = "ProjectWithPackage", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = false, + }; + + libraryProject.PackageReferences.Add(new TestPackageReference("Microsoft.Build", "17.3.1")); + + var consumerProject = new TestProject() + { + Name = "ConsumerProject", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = true, + }; + + consumerProject.AdditionalProperties.Add("CopyDebugSymbolFilesFromPackages", enableCopyDebugSymbolFilesFromPackages); + consumerProject.AdditionalProperties.Add("CopyDocumentationFilesFromPackages", enableDocumentationFilesFromPackages); + + consumerProject.ReferencedProjects.Add(libraryProject); + + string testPath = enableCopyDebugSymbolFilesFromPackages + enableDocumentationFilesFromPackages; + TestAsset testAsset = _testAssetsManager.CreateTestProject(consumerProject, consumerProject.Name, identifier: testPath); + + var buildCommand = new BuildCommand(testAsset); + + buildCommand + .Execute() + .Should() + .Pass(); + + var outputDirectory = buildCommand.GetOutputDirectory(libraryProject.TargetFrameworks); + + HelperCheckPdbAndDocumentation(outputDirectory, "Microsoft.Build", enableCopyDebugSymbolFilesFromPackages, enableDocumentationFilesFromPackages); + } + + [WindowsOnlyTheory] + [InlineData("true", "true")] + [InlineData("true", "false")] + [InlineData("false", "true")] + [InlineData("false", "false")] + public void It_places_package_pdb_and_xml_files_in_publish_directory(string enableCopyDebugSymbolFilesFromPackages, string enableDocumentationFilesFromPackages) + { + var testProject = new TestProject() + { + Name = "DesktopPublishPackageWithPdbAndXml", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = true, + }; + + testProject.PackageReferences.Add(new TestPackageReference("Microsoft.Build", "17.3.1")); + + testProject.AdditionalProperties.Add("CopyDebugSymbolFilesFromPackages", enableCopyDebugSymbolFilesFromPackages); + testProject.AdditionalProperties.Add("CopyDocumentationFilesFromPackages", enableDocumentationFilesFromPackages); + + string testPath = enableCopyDebugSymbolFilesFromPackages + enableDocumentationFilesFromPackages; + TestAsset testAsset = _testAssetsManager.CreateTestProject(testProject, testProject.Name, identifier: testPath); + + var buildCommand = new BuildCommand(testAsset); + + buildCommand + .Execute() + .Should() + .Pass(); + + var publishCommand = new PublishCommand(testAsset); + + publishCommand + .Execute() + .Should() + .Pass(); + + var publishDirectory = publishCommand.GetOutputDirectory(testProject.TargetFrameworks); + + HelperCheckPdbAndDocumentation(publishDirectory, "Microsoft.Build", enableCopyDebugSymbolFilesFromPackages, enableDocumentationFilesFromPackages); + } + + [WindowsOnlyFact] + public void It_places_package_xml_files_in_output_directory_but_not_in_publish() + { + var testProject = new TestProject() + { + Name = "DesktopPackageWithXmlNotPublished", + TargetFrameworks = ToolsetInfo.CurrentTargetFramework, + IsExe = true, + }; + + testProject.PackageReferences.Add(new TestPackageReference("Microsoft.Build", "17.3.1")); + + testProject.AdditionalProperties.Add("PublishReferencesDocumentationFiles", "false"); + testProject.AdditionalProperties.Add("CopyDocumentationFilesFromPackages", "true"); + + TestAsset testAsset = _testAssetsManager.CreateTestProject(testProject, testProject.Name); + + var buildCommand = new BuildCommand(testAsset); + + buildCommand + .Execute() + .Should() + .Pass(); + + var publishCommand = new PublishCommand(testAsset); + + publishCommand + .Execute() + .Should() + .Pass(); + + var outputDirectory = buildCommand.GetOutputDirectory(testProject.TargetFrameworks); + + var publishDirectory = publishCommand.GetOutputDirectory(testProject.TargetFrameworks); + + outputDirectory.Should().HaveFile("Microsoft.Build.xml"); + publishDirectory.Should().NotHaveFile("Microsoft.Build.xml"); + } + + void HelperCheckPdbAndDocumentation( + DirectoryInfo directoryInfo, + string packageName, + string enableCopyDebugSymbolFilesFromPackages, + string enableDocumentationFilesFromPackages) + { + string assemblyFile = packageName + ".dll"; + string pdbFile = packageName + ".pdb"; + string docFile = packageName + ".xml"; + + ShouldHave(assemblyFile); + if (enableCopyDebugSymbolFilesFromPackages == "true") + { + ShouldHave(pdbFile); + } + else + { + ShouldNotHave(pdbFile); + } + + if (enableDocumentationFilesFromPackages == "true") + { + ShouldHave(docFile); + } + else + { + ShouldNotHave(docFile); + } + + void ShouldHave(string file) => directoryInfo.Should().HaveFile(file); + + void ShouldNotHave(string file) => directoryInfo.Should().NotHaveFile(file); + } } }