Implements support for Embedded Portable PDB (#10199)

* Implements support for Embedded Portable PDB

* Remove unnecessary asserts

* Add BlobContentId ctor that takes byte[]

* Add signature to the embedded blob

src/System.Reflection.Metadata/System.Reflection.Metadata.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.24720.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.Reflection.Metadata", "src\System.Reflection.Metadata.csproj", "{F3E433C8-352F-4944-BF7F-765CE435370D}"
 EndProject

src/System.Reflection.Metadata/specs/PE-COFF.md

@@ -29,10 +29,30 @@ The associated .pdb file may not exist at the path indicated by Path field. If i
 
 If the containing PE/COFF file is deterministic the Guid field above and DateTimeStamp field of the directory entry are  calculated deterministically based solely on the content of the associated .pdb file. Otherwise the value of Guid is random and the value of DateTimeStamp indicates the time and date that the debug data was created.
 
-*Version Major=0x0100, Minor=0x504d* of the data format has the same structure as above. The Age shall be 1. The format of the associated .pdb file is Portable PDB. Together 16B of the Guid concatenated with 4B of the TimeDateStamp field of the entry form a PDB ID that should be used to match the PE/COFF image with the associated PDB (instead of Guid and Age). Matching PDB ID is stored in the #Pdb stream of the .pdb file.
+*Version Major=any, Minor=0x504d* of the data format has the same structure as above. The Age shall be 1. The format of the associated .pdb file is Portable PDB. The Major version specified in the entry indicates the version of the Portable PDB format. Together 16B of the Guid concatenated with 4B of the TimeDateStamp field of the entry form a PDB ID that should be used to match the PE/COFF image with the associated PDB (instead of Guid and Age). Matching PDB ID is stored in the #Pdb stream of the .pdb file.
 
 ### Deterministic Debug Directory Entry (type 16)
 
 The entry doesn't have any data associated with it. All fields of the entry, but Type shall be zero.
 
 Presence of this entry indicates that the containing PE/COFF file is deterministic. 
+
+### Embedded Portable PDB Debug Directory Entry (type 17)
+
+Declares that debugging information is embedded in the PE file at location specified by PointerToRawData. 
+
+*Version Major=any, Minor=0x0100* of the data format:
+
+| Offset | Size           | Field            | Description                                           |
+|:-------|:---------------|:-----------------|-------------------------------------------------------|
+| 0      | 4              | Signature        | 0x4D 0x50 0x44 0x42                                   |
+| 4      | 4              | UncompressedSize | The size of decompressed Portable PDB image           |
+| 8      | SizeOfData - 8 | PortablePdbImage | Portable PDB image compressed using Deflate algorithm | 
+
+If this entry is present and the reader recognizes this entry the debugging information for the PE file shall be read from the embedded data. If a CodeView entry is also present it shall be ignored. 
+
+> Note: Including both entries enables a tool that does not recognize Embedded Portable PDB entry to locate debug infomration as long as it is also available in a file specified in CodeView entry. Such file can be created by extracting the embedded Portable PDB image to a separate file.
+
+The Major version specified in the entry indicates the version of the Portable PDB format. The Minor version indicates the version of the Embedded Portable PDB data format.
+
+The value of Stamp field in the entry shall be 0.

src/System.Reflection.Metadata/src/Resources/Strings.resx

@@ -348,6 +348,9 @@
   <data name="UnexpectedCodeViewDataSignature" xml:space="preserve">
     <value>Unexpected CodeView data signature value.</value>
   </data>
+  <data name="UnexpectedEmbeddedPortablePdbDataSignature" xml:space="preserve">
+    <value>Unexpected Embedded Portable PDB data signature value.</value>
+  </data>
   <data name="InvalidPathPadding" xml:space="preserve">
     <value>The path must be padded with NUL characters.</value>
   </data>
@@ -426,6 +429,9 @@
   <data name="HashTooShort" xml:space="preserve">
     <value>Hash must be at least {0}B long.</value>
   </data>
+  <data name="UnexpectedArrayLength" xml:space="preserve">
+    <value>Expected array of length {0}.</value>
+  </data>
   <data name="ValueMustBeMultiple" xml:space="preserve">
     <value>Value must be multiple of {0}.</value>
   </data>
@@ -441,4 +447,13 @@
   <data name="RowCountOutOfRange" xml:space="preserve">
     <value>Row count specified for table index {0} is out of allowed range.</value>
   </data>
+  <data name="SizeMismatch" xml:space="preserve">
+    <value>Declared size doesn't correspond to the actual size.</value>
+  </data>
+  <data name="DataTooBig" xml:space="preserve">
+    <value>Data too big to fit in memory.</value>
+  </data>
+  <data name="UnsupportedFormatVersion" xml:space="preserve">
+    <value>Unsupported format version: {0}</value>
+  </data>
 </root>

src/System.Reflection.Metadata/src/System.Reflection.Metadata.csproj

@@ -55,6 +55,7 @@
     <Compile Include="System\Reflection\Internal\Utilities\DecimalUtilities.cs" />
     <Compile Include="System\Reflection\Internal\Utilities\EnumerableExtensions.cs" />
     <Compile Include="System\Reflection\Metadata\Ecma335\CustomAttributeDecoder.cs" />
+    <Compile Include="System\Reflection\Metadata\PortablePdb\PortablePdbVersions.cs" />
     <Compile Include="System\Reflection\Metadata\Signatures\CustomAttributeNamedArgument.cs" />
     <Compile Include="System\Reflection\Metadata\Signatures\CustomAttributeTypedArgument.cs" />
     <Compile Include="System\Reflection\Metadata\Signatures\CustomAttributeValue.cs" />

src/System.Reflection.Metadata/src/System/Reflection/Metadata/BlobContentId.cs

@@ -10,6 +10,8 @@ namespace System.Reflection.Metadata
 {
     public struct BlobContentId
     {
+        private const int Size = BlobUtilities.SizeOfGuid + sizeof(uint);
+
         public Guid Guid { get; }
         public uint Stamp { get; }
 
@@ -19,6 +21,31 @@ public BlobContentId(Guid guid, uint stamp)
             Stamp = stamp;
         }
 
+        public BlobContentId(ImmutableArray<byte> id) 
+            : this(ImmutableByteArrayInterop.DangerousGetUnderlyingArray(id))
+        {
+        }
+
+        public unsafe BlobContentId(byte[] id)
+        {
+            if (id == null)
+            {
+                throw new ArgumentNullException(nameof(id));
+            }
+
+            if (id.Length != Size)
+            {
+                throw new ArgumentException(SR.Format(SR.UnexpectedArrayLength, Size), nameof(id));
+            }
+
+            fixed (byte* ptr = id)
+            {
+                var reader = new BlobReader(ptr, id.Length);
+                Guid = reader.ReadGuid();
+                Stamp = reader.ReadUInt32();
+            }
+        }
+
         public bool IsDefault => Guid == default(Guid) && Stamp == 0;
 
         public static BlobContentId FromHash(ImmutableArray<byte> hashCode)

src/System.Reflection.Metadata/src/System/Reflection/Metadata/Ecma335/PortablePdbBuilder.cs

@@ -14,8 +14,8 @@ namespace System.Reflection.Metadata.Ecma335
     /// </summary>
     public sealed class PortablePdbBuilder
     {
-        public string MetadataVersion => "PDB v1.0";
-        public ushort FormatVersion => 0x0100;
+        public string MetadataVersion => PortablePdbVersions.DefaultMetadataVersion;
+        public ushort FormatVersion => PortablePdbVersions.DefaultFormatVersion;
 
         private Blob _pdbIdBlob;
         private readonly MethodDefinitionHandle _entryPoint;

src/System.Reflection.Metadata/src/System/Reflection/Metadata/MetadataReader.cs

@@ -1191,7 +1191,6 @@ public ExportedType GetExportedType(ExportedTypeHandle handle)
 
         public CustomAttributeHandleCollection GetCustomAttributes(EntityHandle handle)
         {
-            Debug.Assert(!handle.IsNil);
             return new CustomAttributeHandleCollection(this, handle);
         }
 
@@ -1418,7 +1417,6 @@ public CustomDebugInformation GetCustomDebugInformation(CustomDebugInformationHa
 
         public CustomDebugInformationHandleCollection GetCustomDebugInformation(EntityHandle handle)
         {
-            Debug.Assert(!handle.IsNil);
             return new CustomDebugInformationHandleCollection(this, handle);
         }
 

src/System.Reflection.Metadata/src/System/Reflection/Metadata/PortablePdb/HandleCollections.Debug.cs

@@ -745,7 +745,6 @@ internal CustomDebugInformationHandleCollection(MetadataReader reader)
         internal CustomDebugInformationHandleCollection(MetadataReader reader, EntityHandle handle)
         {
             Debug.Assert(reader != null);
-            Debug.Assert(!handle.IsNil);
 
             _reader = reader;
             reader.CustomDebugInformationTable.GetRange(handle, out _firstRowId, out _lastRowId);

src/System.Reflection.Metadata/src/System/Reflection/Metadata/PortablePdb/PortablePdbVersions.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Reflection.Metadata
+{
+    internal static class PortablePdbVersions
+    {
+        /// <summary>
+        /// Version of Portable PDB format emitted by the writer by default. Metadata version string.
+        /// </summary>
+        internal const string DefaultMetadataVersion = "PDB v1.0";
+
+        /// <summary>
+        /// Version of Portable PDB format emitted by the writer by default.
+        /// </summary>
+        internal const ushort DefaultFormatVersion = 0x0100;
+
+        /// <summary>
+        /// Minimal supported version of Portable PDB format.
+        /// </summary>
+        internal const ushort MinFormatVersion = 0x0100;
+
+        /// <summary>
+        /// Minimal supported version of Embedded Portable PDB blob.
+        /// </summary>
+        internal const ushort MinEmbeddedVersion = 0x0100;
+
+        /// <summary>
+        /// Version of Embedded Portable PDB blob format emitted by the writer by default.
+        /// </summary>
+        internal const ushort DefaultEmbeddedVersion = 0x0100;
+
+        /// <summary>
+        /// Minimal version of the Embedded Portable PDB blob that the current reader can't interpret.
+        /// </summary>
+        internal const ushort MinUnsupportedEmbeddedVersion = 0x0200;
+
+        internal const uint DebugDirectoryEmbeddedSignature = 0x4244504d;
+
+        internal static uint DebugDirectoryEntryVersion(ushort portablePdbVersion) => 'P' << 24 | 'M' << 16 | (uint)portablePdbVersion;
+        internal static uint DebugDirectoryEmbeddedVersion(ushort portablePdbVersion) => (uint)DefaultEmbeddedVersion << 16 | portablePdbVersion;
+        internal static string Format(ushort version) => (version >> 8) + "." + (version & 0xff);
+    }
+}

src/System.Reflection.Metadata/src/System/Reflection/Metadata/TypeSystem/HandleCollections.TypeSystem.cs

@@ -243,7 +243,6 @@ internal CustomAttributeHandleCollection(MetadataReader reader)
         internal CustomAttributeHandleCollection(MetadataReader reader, EntityHandle handle)
         {
             Debug.Assert(reader != null);
-            Debug.Assert(!handle.IsNil);
 
             _reader = reader;
             reader.CustomAttributeTable.GetAttributeRange(handle, out _firstRowId, out _lastRowId);

src/System.Reflection.Metadata/src/System/Reflection/PortableExecutable/DebugDirectory/DebugDirectoryBuilder.cs

@@ -3,6 +3,8 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
 using System.Reflection.Metadata;
 
 namespace System.Reflection.PortableExecutable
@@ -51,14 +53,55 @@ public void AddCodeViewEntry(
 
             AddEntry(
                 type: DebugDirectoryEntryType.CodeView,
+                version: (portablePdbVersion == 0) ? 0 : PortablePdbVersions.DebugDirectoryEntryVersion(portablePdbVersion),
                 stamp: pdbContentId.Stamp,
-                version: (portablePdbVersion == 0) ? 0 : ('P' << 24 | 'M' << 16 | (uint)portablePdbVersion),
                 dataSize: dataSize);
         }
 
         public void AddReproducibleEntry()
         {
-            AddEntry(type: DebugDirectoryEntryType.Reproducible, stamp: 0, version: 0);
+            AddEntry(type: DebugDirectoryEntryType.Reproducible, version: 0, stamp: 0);
+        }
+
+        public void AddEmbeddedPortablePdbEntry(BlobBuilder debugMetadata, ushort portablePdbVersion)
+        {
+            if (debugMetadata == null)
+            {
+                Throw.ArgumentNull(nameof(debugMetadata));
+            }
+
+            int dataSize = WriteEmbeddedPortablePdbData(_dataBuilder, debugMetadata);
+
+            AddEntry(
+                type: DebugDirectoryEntryType.EmbeddedPortablePdb, 
+                version: PortablePdbVersions.DebugDirectoryEmbeddedVersion(portablePdbVersion),
+                stamp: 0,
+                dataSize: dataSize);
+        }
+
+        private static int WriteEmbeddedPortablePdbData(BlobBuilder builder, BlobBuilder debugMetadata)
+        {
+            int start = builder.Count;
+
+            // header (signature, decompressed size):
+            builder.WriteUInt32(PortablePdbVersions.DebugDirectoryEmbeddedSignature);
+            builder.WriteInt32(debugMetadata.Count);
+
+            // compressed data:
+            var compressed = new MemoryStream();
+            using (var deflate = new DeflateStream(compressed, CompressionLevel.Optimal, leaveOpen: true))
+            {
+                foreach (var blob in debugMetadata.GetBlobs())
+                {
+                    var segment = blob.GetBytes();
+                    deflate.Write(segment.Array, segment.Offset, segment.Count);
+                }
+            }
+
+            // TODO: avoid multiple copies:
+            builder.WriteBytes(compressed.ToArray());
+
+            return builder.Count - start;
         }
 
         private static int WriteCodeViewData(BlobBuilder builder, string pdbPath, Guid pdbGuid)

src/System.Reflection.Metadata/src/System/Reflection/PortableExecutable/DebugDirectory/DebugDirectoryEntryType.cs

@@ -43,5 +43,17 @@ public enum DebugDirectoryEntryType
         /// </para>
         /// </remarks>
         Reproducible = 16,
+
+        /// <summary>
+        /// The entry points to a blob containing Embedded Portable PDB.
+        /// </summary>
+        /// <remarks>
+        /// The Embedded Portable PDB blob has the following format:
+        /// 
+        /// blob ::= uncompressed-size data
+        /// 
+        /// Data spans the remainder of the blob and contains a Deflate-compressed Portable PDB.
+        /// </remarks>
+        EmbeddedPortablePdb = 17,
     }
 }