Add generated semantic search API list

Cleanup

src/Tools/SemanticSearch/BuildTask/GenerateFilteredReferenceAssembliesTask.cs

@@ -13,6 +13,7 @@
 using System.Reflection.Metadata.Ecma335;
 using System.Reflection.PortableExecutable;
 using System.Security.Cryptography;
+using System.Text;
 using System.Text.RegularExpressions;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
@@ -49,8 +50,7 @@ public sealed class GenerateFilteredReferenceAssembliesTask : Task
         \s*
         (?<Inclusion>[+|-]?)
         ((?<Kinds>[A-Za-z]+):)?
-        (?<MetadataName>[^#]*)
-        ([#].*)?
+        (?<MetadataName>.*)
         $
         """, RegexOptions.Singleline | RegexOptions.IgnorePatternWhitespace);
 
@@ -63,6 +63,9 @@ public sealed class GenerateFilteredReferenceAssembliesTask : Task
     [Required]
     public string OutputDir { get; private set; } = null!;
 
+    [Required]
+    public string ApisDir { get; private set; } = null!;
+
     public override bool Execute()
     {
         try
@@ -106,6 +109,8 @@ internal void ExecuteImpl(IEnumerable<(string apiSpecPath, IReadOnlyList<string>
             }
 
             var peImageBuffer = File.ReadAllBytes(originalReferencePath);
+
+            var baseline = new List<string>();
             Rewrite(peImageBuffer, patterns.ToImmutableArray());
 
             try
@@ -116,15 +121,71 @@ internal void ExecuteImpl(IEnumerable<(string apiSpecPath, IReadOnlyList<string>
             {
                 // Another instance of the task might already be writing the content. 
                 Log.LogMessage($"Output file '{filteredReferencePath}' already exists.");
+                return;
+            }
+
+            WriteApis(Path.Combine(ApisDir, assemblyName + ".txt"), peImageBuffer);
+        }
+    }
+
+    internal void WriteApis(string baselineFilePath, byte[] peImage)
+    {
+        using var readableStream = new MemoryStream(peImage, writable: false);
+        var metadataRef = MetadataReference.CreateFromStream(readableStream);
+        var compilation = CSharpCompilation.Create("Metadata", references: [metadataRef]);
+
+        // Collect all externally accessible metadata member definitions:
+        var types = new List<INamedTypeSymbol>();
+        var methods = new List<IMethodSymbol>();
+        var fields = new List<IFieldSymbol>();
+        GetAllMembers(compilation, types, methods, fields,
+            filter: s => s is { MetadataToken: not 0, DeclaredAccessibility: Accessibility.Public or Accessibility.Protected or Accessibility.ProtectedOrInternal });
+
+        var apis = new List<string>();
+        apis.AddRange(((IEnumerable<ISymbol>)types).Concat(methods).Concat(fields).Select(GetDocumentationCommentSymbolName));
+
+        // Doc ids start with "X:" prefix, where X is member kind ('T', 'M' or 'F'):
+        apis.Sort(static (x, y) => x.AsSpan()[2..].CompareTo(y.AsSpan()[2..], StringComparison.Ordinal));
+
+        var newContent = string.Join("\r\n", apis);
+
+        string currentContent;
+        try
+        {
+            currentContent = File.ReadAllText(baselineFilePath, Encoding.UTF8);
+        }
+        catch (Exception)
+        {
+            currentContent = "";
+        }
+
+        if (currentContent != newContent)
+        {
+            try
+            {
+                File.WriteAllText(baselineFilePath, newContent);
+                Log.LogMessage($"Baseline updated: '{baselineFilePath}'");
+            }
+            catch (Exception e)
+            {
+                Log.LogError($"Error updating baseline '{baselineFilePath}': {e.Message}");
             }
         }
+        else
+        {
+            Log.LogMessage($"Baseline not updated '{baselineFilePath}'");
+        }
     }
 
     internal static void ParseApiPatterns(IReadOnlyList<string> lines, List<(string message, int line)> errors, List<ApiPattern> patterns)
     {
         for (var i = 0; i < lines.Count; i++)
         {
             var line = lines[i];
+            if (line.TrimStart().StartsWith("#"))
+            {
+                continue;
+            }
 
             var match = s_lineSyntax.Match(line);
             if (!match.Success)
@@ -197,7 +258,8 @@ internal static void GetAllMembers(
         Compilation compilation,
         List<INamedTypeSymbol> types,
         List<IMethodSymbol> methods,
-        List<IFieldSymbol> fields)
+        List<IFieldSymbol> fields,
+        Func<ISymbol, bool> filter)
     {
         Recurse(compilation.GlobalNamespace.GetMembers());
 
@@ -208,22 +270,22 @@ void Recurse(IEnumerable<ISymbol> members)
                 switch (member)
                 {
                     case INamedTypeSymbol type:
-                        if (type.MetadataToken != 0)
+                        if (filter(member))
                         {
                             types.Add(type);
                             Recurse(type.GetMembers());
                         }
                         break;
 
                     case IMethodSymbol method:
-                        if (method.MetadataToken != 0)
+                        if (filter(member))
                         {
                             methods.Add(method);
                         }
                         break;
 
                     case IFieldSymbol field:
-                        if (field.MetadataToken != 0)
+                        if (filter(member))
                         {
                             fields.Add(field);
                         }
@@ -237,12 +299,16 @@ void Recurse(IEnumerable<ISymbol> members)
         }
     }
 
-    private static bool IsIncluded(ISymbol symbol, ImmutableArray<ApiPattern> patterns)
+    private static string GetDocumentationCommentSymbolName(ISymbol symbol)
     {
         var id = symbol.GetDocumentationCommentId();
         Debug.Assert(id is [_, ':', ..]);
-        id = id[2..];
+        return id[2..];
+    }
 
+    private static bool IsIncluded(ISymbol symbol, ImmutableArray<ApiPattern> patterns)
+    {
+        var docName = GetDocumentationCommentSymbolName(symbol);
         var kind = GetKindFlags(symbol);
 
         // Type symbols areconsidered excluded by default.
@@ -251,7 +317,7 @@ private static bool IsIncluded(ISymbol symbol, ImmutableArray<ApiPattern> patter
 
         foreach (var pattern in patterns)
         {
-            if ((pattern.SymbolKinds & kind) == kind && pattern.MetadataNamePattern.IsMatch(id))
+            if ((pattern.SymbolKinds & kind) == kind && pattern.MetadataNamePattern.IsMatch(docName))
             {
                 isIncluded = pattern.IsIncluded;
             }
@@ -285,7 +351,7 @@ internal static unsafe void Rewrite(byte[] peImage, ImmutableArray<ApiPattern> p
         var types = new List<INamedTypeSymbol>();
         var methods = new List<IMethodSymbol>();
         var fields = new List<IFieldSymbol>();
-        GetAllMembers(compilation, types, methods, fields);
+        GetAllMembers(compilation, types, methods, fields, filter: s => s.MetadataToken != 0);
 
         // Update visibility flags:
         using var writableStream = new MemoryStream(peImage, writable: true);

src/Tools/SemanticSearch/ReferenceAssemblies/ApiSet/Microsoft.CodeAnalysis.CSharp.txt

@@ -1 +1,9 @@
-
++T:*
+-T:Microsoft.CodeAnalysis.CSharp.CSharpCommandLine*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.#ctor*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.CommonWithCryptoKeyContainer*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.CommonWithCryptoKeyFile*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.CommonWithCryptoPublicKey*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.WithCryptoKeyContainer*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.WithCryptoKeyFile*
+-M:Microsoft.CodeAnalysis.CSharp.CSharpCompilationOptions.WithCryptoPublicKey*

src/Tools/SemanticSearch/ReferenceAssemblies/ApiSet/Microsoft.CodeAnalysis.txt

@@ -1,5 +1,23 @@
 +T:*
--M:Microsoft.CodeAnalysis.MetadataReference.CreateFromFile*
+-M:Microsoft.CodeAnalysis.MetadataReference.Create*
+-M:Microsoft.CodeAnalysis.AssemblyMetadata.Create*
+-M:Microsoft.CodeAnalysis.ModuleMetadata.Create*
 -T:Microsoft.CodeAnalysis.FileSystemExtensions
--T:Microsoft.CodeAnalysis.CommandLineParser
--T:Microsoft.CodeAnalysis.DesktopStrongNameProvider
+-T:Microsoft.CodeAnalysis.CommandLine*
+-T:Microsoft.CodeAnalysis.DesktopStrongNameProvider
+-M:Microsoft.CodeAnalysis.CompilationOptions.CommonWithCryptoKeyContainer*
+-M:Microsoft.CodeAnalysis.CompilationOptions.CommonWithCryptoKeyFile*
+-M:Microsoft.CodeAnalysis.CompilationOptions.CommonWithCryptoPublicKey*
+-M:Microsoft.CodeAnalysis.CompilationOptions.WithCryptoKeyContainer*
+-M:Microsoft.CodeAnalysis.CompilationOptions.WithCryptoKeyFile*
+-M:Microsoft.CodeAnalysis.CompilationOptions.WithCryptoPublicKey*
+-M:Microsoft.CodeAnalysis.Compilation.Emit*
+-T:Microsoft.CodeAnalysis.Emit.*
+-M:Microsoft.CodeAnalysis.Diagnostics.AnalyzerFileReference.#ctor*
+-M:Microsoft.CodeAnalysis.Diagnostics.AnalyzerImageReference.#ctor*
+-T:Microsoft.CodeAnalysis.RuleSet
+-M:Microsoft.CodeAnalysis.SourceReferenceResolver.*
+-M:Microsoft.CodeAnalysis.SourceFileResolver.*
+-M:Microsoft.CodeAnalysis.XmlReferenceResolver.*
+-M:Microsoft.CodeAnalysis.MetadataReferenceResolver.*
+-M:Microsoft.CodeAnalysis.StrongNameProvider.*

src/Tools/SemanticSearch/ReferenceAssemblies/ApiSet/System.Runtime.txt

@@ -3,13 +3,17 @@
 -T:System.AppDomain*
 -T:System.AssemblyLoad*
 -T:System.AppContext
+-T:System.CodeDom*
+-T:System.Diagnostics.Debug.*
 -TM:System.Environment.*
 +M:System.Environment.get_CurrentManagedThreadId
 +M:System.Environment.get_NewLine
 -T:System.EnvironmentVariableTarget
 -T:System.GC*
+-T:System.IntPtr
 -T:System.LoaderOptimization*
 -T:System.MarshalByRefObject
+-T:System.ModuleHandle
 -T:System.MTAThreadAttribute
 -T:System.STAThreadAttribute
 -T:System.ThreadStaticAttribute
@@ -19,27 +23,15 @@
 +M:System.Type.Name
 +M:System.Type.FullName
 +M:System.Type.AssemblyQualifiedName
--T:System.IO.*
-+T:System.IO.BinaryReader
-+T:System.IO.BinaryWriter
-+T:System.IO.BufferedStream
-+T:System.IO.EndOfStreamException
-+T:System.IO.InvalidDataException
-+T:System.IO.MemoryStream
-+T:System.IO.Stream
-+T:System.IO.StreamReader
-+T:System.IO.StreamWriter
-+T:System.IO.StringReader
-+T:System.IO.StringWriter
-+T:System.IO.TextReader
-+T:System.IO.TextWriter
 
+-T:System.IO.*
 -T:System.Net.*
 -T:System.Reflection.*
 -T:System.Resources.*
 
 -T:System.Runtime.*
 +T:System.Runtime.CompilerServices.*
+-T:System.Runtime.CompilerServices.MethodImplOptions
 
 -T:System.Security.*