From 124506731a9a28c05eb420188a32301e1d497101 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 3 Mar 2021 17:24:19 -0700 Subject: [PATCH 1/2] Add ability to generate enum declarations from docs --- src/Microsoft.Windows.CsWin32/Generator.cs | 58 +---- .../GeneratorUtilities.cs | 93 ++++++++ src/ScrapeDocs/DocEnum.cs | 105 ++++++++- src/ScrapeDocs/Program.cs | 201 ++++++++++++++++-- src/ScrapeDocs/ScrapeDocs.csproj | 12 ++ 5 files changed, 381 insertions(+), 88 deletions(-) create mode 100644 src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs diff --git a/src/Microsoft.Windows.CsWin32/Generator.cs b/src/Microsoft.Windows.CsWin32/Generator.cs index 0415e48e..8080a6ee 100644 --- a/src/Microsoft.Windows.CsWin32/Generator.cs +++ b/src/Microsoft.Windows.CsWin32/Generator.cs @@ -3426,62 +3426,8 @@ private MarshalAsAttribute ToMarshalAsAttribute(BlobHandle blobHandle) return ma; } - private ExpressionSyntax ToExpressionSyntax(Constant constant) - { - var blobReader = this.mr.GetBlobReader(constant.Value); - return constant.TypeCode switch - { - ConstantTypeCode.Boolean => blobReader.ReadBoolean() ? LiteralExpression(SyntaxKind.TrueLiteralExpression) : LiteralExpression(SyntaxKind.FalseLiteralExpression), - ConstantTypeCode.Char => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadChar())), - ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadSByte())), - ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadByte())), - ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt16())), - ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt16())), - ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt32())), - ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt32())), - ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt64())), - ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt64())), - ConstantTypeCode.Single => FloatExpression(blobReader.ReadSingle()), - ConstantTypeCode.Double => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadDouble())), - ConstantTypeCode.String => blobReader.ReadConstant(constant.TypeCode) is string value ? LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(value)) : LiteralExpression(SyntaxKind.NullLiteralExpression), - ConstantTypeCode.NullReference => LiteralExpression(SyntaxKind.NullLiteralExpression), - _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), - }; + private ExpressionSyntax ToExpressionSyntax(Constant constant) => GeneratorUtilities.ToExpressionSyntax(this.mr, constant); - static ExpressionSyntax FloatExpression(float value) - { - return - float.IsPositiveInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.PositiveInfinity))) : - float.IsNegativeInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NegativeInfinity))) : - float.IsNaN(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NaN))) : - LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(value)); - } - } - - private ExpressionSyntax ToHexExpressionSyntax(Constant constant) - { - var blobReader = this.mr.GetBlobReader(constant.Value); - var blobReader2 = this.mr.GetBlobReader(constant.Value); - return constant.TypeCode switch - { - ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadSByte()), blobReader2.ReadSByte())), - ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadByte()), blobReader2.ReadByte())), - ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt16()), blobReader2.ReadInt16())), - ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt16()), blobReader2.ReadUInt16())), - ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt32()), blobReader2.ReadInt32())), - ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt32()), blobReader2.ReadUInt32())), - ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt64()), blobReader2.ReadInt64())), - ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt64()), blobReader2.ReadUInt64())), - _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), - }; - - unsafe string ToHex(T value) - where T : unmanaged - { - int fullHexLength = sizeof(T) * 2; - string hex = string.Format(CultureInfo.InvariantCulture, "0x{0:X" + fullHexLength + "}", value); - return hex; - } - } + private ExpressionSyntax ToHexExpressionSyntax(Constant constant) => GeneratorUtilities.ToHexExpressionSyntax(this.mr, constant); } } diff --git a/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs b/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs new file mode 100644 index 00000000..4091ecf7 --- /dev/null +++ b/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.Reflection.Metadata; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using static Microsoft.CodeAnalysis.CSharp.SyntaxFactory; + +internal static class GeneratorUtilities +{ + internal static PredefinedTypeSyntax ToTypeOfConstant(MetadataReader mr, Constant constant) + { + var blobReader = mr.GetBlobReader(constant.Value); + SyntaxKind keyword = constant.TypeCode switch + { + ConstantTypeCode.Boolean => SyntaxKind.BoolKeyword, + ConstantTypeCode.Char => SyntaxKind.CharKeyword, + ConstantTypeCode.SByte => SyntaxKind.SByteKeyword, + ConstantTypeCode.Byte => SyntaxKind.ByteKeyword, + ConstantTypeCode.Int16 => SyntaxKind.ShortKeyword, + ConstantTypeCode.UInt16 => SyntaxKind.UShortKeyword, + ConstantTypeCode.Int32 => SyntaxKind.IntKeyword, + ConstantTypeCode.UInt32 => SyntaxKind.UIntKeyword, + ConstantTypeCode.Int64 => SyntaxKind.LongKeyword, + ConstantTypeCode.UInt64 => SyntaxKind.ULongKeyword, + ConstantTypeCode.Single => SyntaxKind.FloatKeyword, + ConstantTypeCode.Double => SyntaxKind.DoubleKeyword, + ConstantTypeCode.String => SyntaxKind.StringKeyword, + _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), + }; + return PredefinedType(Token(keyword)); + } + + internal static ExpressionSyntax ToExpressionSyntax(MetadataReader mr, Constant constant) + { + var blobReader = mr.GetBlobReader(constant.Value); + return constant.TypeCode switch + { + ConstantTypeCode.Boolean => blobReader.ReadBoolean() ? LiteralExpression(SyntaxKind.TrueLiteralExpression) : LiteralExpression(SyntaxKind.FalseLiteralExpression), + ConstantTypeCode.Char => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadChar())), + ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadSByte())), + ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadByte())), + ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt16())), + ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt16())), + ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt32())), + ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt32())), + ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt64())), + ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt64())), + ConstantTypeCode.Single => FloatExpression(blobReader.ReadSingle()), + ConstantTypeCode.Double => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadDouble())), + ConstantTypeCode.String => blobReader.ReadConstant(constant.TypeCode) is string value ? LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(value)) : LiteralExpression(SyntaxKind.NullLiteralExpression), + ConstantTypeCode.NullReference => LiteralExpression(SyntaxKind.NullLiteralExpression), + _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), + }; + + static ExpressionSyntax FloatExpression(float value) + { + return + float.IsPositiveInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.PositiveInfinity))) : + float.IsNegativeInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NegativeInfinity))) : + float.IsNaN(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NaN))) : + LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(value)); + } + } + + internal static ExpressionSyntax ToHexExpressionSyntax(MetadataReader mr, Constant constant) + { + var blobReader = mr.GetBlobReader(constant.Value); + var blobReader2 = mr.GetBlobReader(constant.Value); + return constant.TypeCode switch + { + ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadSByte()), blobReader2.ReadSByte())), + ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadByte()), blobReader2.ReadByte())), + ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt16()), blobReader2.ReadInt16())), + ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt16()), blobReader2.ReadUInt16())), + ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt32()), blobReader2.ReadInt32())), + ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt32()), blobReader2.ReadUInt32())), + ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt64()), blobReader2.ReadInt64())), + ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt64()), blobReader2.ReadUInt64())), + _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), + }; + + unsafe string ToHex(T value) + where T : unmanaged + { + int fullHexLength = sizeof(T) * 2; + string hex = string.Format(CultureInfo.InvariantCulture, "0x{0:X" + fullHexLength + "}", value); + return hex; + } + } +} diff --git a/src/ScrapeDocs/DocEnum.cs b/src/ScrapeDocs/DocEnum.cs index d28e7988..574e5f86 100644 --- a/src/ScrapeDocs/DocEnum.cs +++ b/src/ScrapeDocs/DocEnum.cs @@ -4,18 +4,26 @@ namespace ScrapeDocs { using System.Collections.Generic; + using System.Linq; + using System.Reflection.Metadata; + using Microsoft.CodeAnalysis.CSharp; + using Microsoft.CodeAnalysis.CSharp.Syntax; + using static GeneratorUtilities; + using static Microsoft.CodeAnalysis.CSharp.SyntaxFactory; internal class DocEnum { - internal DocEnum(bool isFlags, IReadOnlyDictionary memberNamesAndDocs) + private static readonly AttributeListSyntax FlagsAttributeList = AttributeList().AddAttributes(Attribute(IdentifierName("Flags"))); + + internal DocEnum(bool isFlags, IReadOnlyDictionary members) { this.IsFlags = isFlags; - this.MemberNamesAndDocs = memberNamesAndDocs; + this.Members = members; } internal bool IsFlags { get; } - internal IReadOnlyDictionary MemberNamesAndDocs { get; } + internal IReadOnlyDictionary Members { get; } public override bool Equals(object? obj) => this.Equals(obj as DocEnum); @@ -24,10 +32,10 @@ public override int GetHashCode() unchecked { int hash = this.IsFlags ? 1 : 0; - foreach (KeyValuePair entry in this.MemberNamesAndDocs) + foreach (KeyValuePair entry in this.Members) { hash += entry.Key.GetHashCode(); - hash += entry.Value?.GetHashCode() ?? 0; + hash += (int)(entry.Value.Value ?? 0u); } return hash; @@ -46,19 +54,19 @@ public bool Equals(DocEnum? other) return false; } - if (this.MemberNamesAndDocs.Count != other.MemberNamesAndDocs.Count) + if (this.Members.Count != other.Members.Count) { return false; } - foreach (KeyValuePair entry in this.MemberNamesAndDocs) + foreach (KeyValuePair entry in this.Members) { - if (!other.MemberNamesAndDocs.TryGetValue(entry.Key, out string? value)) + if (!other.Members.TryGetValue(entry.Key, out (ulong? Value, string? Doc) value)) { return false; } - if (entry.Value != value) + if (entry.Value.Value != value.Value) { return false; } @@ -66,5 +74,84 @@ public bool Equals(DocEnum? other) return true; } + + internal (string Namespace, EnumDeclarationSyntax Enum)? Emit(string name, MetadataReader mr, HashSet apiClassHandles) + { + if (this.Members.Count == 2 && this.Members.ContainsKey("TRUE") && this.Members.ContainsKey("FALSE")) + { + return null; + } + + PredefinedTypeSyntax? baseType = null; + string? ns = null; + + // Look up values for each constant. + var values = new Dictionary(); + foreach (var item in this.Members) + { + bool found = false; + foreach (FieldDefinitionHandle handle in mr.FieldDefinitions) + { + FieldDefinition fieldDef = mr.GetFieldDefinition(handle); + if (apiClassHandles.Contains(fieldDef.GetDeclaringType()) && mr.StringComparer.Equals(fieldDef.Name, item.Key)) + { + found = true; + Constant constant = mr.GetConstant(fieldDef.GetDefaultValue()); + values.Add(item.Key, this.IsFlags ? ToHexExpressionSyntax(mr, constant) : ToExpressionSyntax(mr, constant)); + baseType ??= ToTypeOfConstant(mr, constant); + ns ??= mr.GetString(mr.GetTypeDefinition(fieldDef.GetDeclaringType()).Namespace); + break; + } + } + + if (!found) + { + // We couldn't find all the constants required. + return null; + } + } + + if (baseType is null || ns is null) + { + // We don't know all the values. + return null; + } + + // Strip the method's declaring interface from the enum name, where applicable. + NameSyntax enumNameSyntax = ParseName(name); + if (enumNameSyntax is QualifiedNameSyntax qname) + { + enumNameSyntax = qname.Right; + } + + EnumDeclarationSyntax enumDecl = EnumDeclaration(Identifier(enumNameSyntax.ToString())) + .AddModifiers(Token(SyntaxKind.PublicKeyword)) + .AddMembers(this.Members + .Select(kv => EnumMemberDeclaration(Identifier(kv.Key)).WithEqualsValue(EqualsValueClause(values[kv.Key]))).ToArray()); + + if (this.IsFlags) + { + // For flags enums, prefer typing as unsigned integers. + baseType = PredefinedType(Token(baseType.Keyword.Kind() switch + { + SyntaxKind.ShortKeyword => SyntaxKind.UShortKeyword, + SyntaxKind.IntKeyword => SyntaxKind.UIntKeyword, + SyntaxKind.LongKeyword => SyntaxKind.ULongKeyword, + _ => baseType.Keyword.Kind(), + })); + } + + if (baseType is not PredefinedTypeSyntax { Keyword: { RawKind: (int)SyntaxKind.IntKeyword } }) + { + enumDecl = enumDecl.AddBaseListTypes(SimpleBaseType(baseType)); + } + + if (this.IsFlags) + { + enumDecl = enumDecl.AddAttributeLists(FlagsAttributeList); + } + + return (ns, enumDecl); + } } } diff --git a/src/ScrapeDocs/Program.cs b/src/ScrapeDocs/Program.cs index 33463852..6646ef8b 100644 --- a/src/ScrapeDocs/Program.cs +++ b/src/ScrapeDocs/Program.cs @@ -13,9 +13,14 @@ namespace ScrapeDocs using System.IO; using System.Linq; using System.Reflection; + using System.Reflection.Metadata; + using System.Reflection.PortableExecutable; using System.Text; using System.Text.RegularExpressions; using System.Threading; + using Microsoft.CodeAnalysis; + using Microsoft.CodeAnalysis.CSharp; + using Microsoft.CodeAnalysis.CSharp.Syntax; using YamlDotNet.RepresentationModel; /// @@ -25,11 +30,12 @@ internal class Program { private static readonly Regex FileNamePattern = new Regex(@"^\w\w-\w+-([\w\-]+)$", RegexOptions.Compiled); private static readonly Regex ParameterHeaderPattern = new Regex(@"^### -param (\w+)", RegexOptions.Compiled); - private static readonly Regex FieldHeaderPattern = new Regex(@"^### -field (\w+)", RegexOptions.Compiled); + private static readonly Regex FieldHeaderPattern = new Regex(@"^### -field (?:\w+\.)*(\w+)", RegexOptions.Compiled); private static readonly Regex ReturnHeaderPattern = new Regex(@"^## -returns", RegexOptions.Compiled); private static readonly Regex RemarksHeaderPattern = new Regex(@"^## -remarks", RegexOptions.Compiled); private static readonly Regex InlineCodeTag = new Regex(@"\(.*)\", RegexOptions.Compiled); private static readonly Regex EnumNameCell = new Regex(@"\]*\>\([\dxa-f]+)\<\/dt\>", RegexOptions.Compiled | RegexOptions.IgnoreCase); private readonly string contentBasePath; private readonly string outputPath; @@ -39,6 +45,8 @@ private Program(string contentBasePath, string outputPath) this.outputPath = outputPath; } + private bool EmitEnums { get; set; } + private static int Main(string[] args) { using var cts = new CancellationTokenSource(); @@ -49,18 +57,19 @@ private static int Main(string[] args) e.Cancel = true; }; - if (args.Length != 2) + if (args.Length < 2) { - Console.Error.WriteLine("USAGE: {0} "); + Console.Error.WriteLine("USAGE: {0} [enums]"); return 1; } string contentBasePath = args[0]; string outputPath = args[1]; + bool emitEnums = args.Length > 2 ? args[2] == "enums" : false; try { - new Program(contentBasePath, outputPath).Worker(cts.Token); + new Program(contentBasePath, outputPath) { EmitEnums = true }.Worker(cts.Token); } catch (OperationCanceledException ex) when (ex.CancellationToken == cts.Token) { @@ -78,7 +87,26 @@ private static void Expect(string? expected, string? actual) } } - private static int AnalyzeEnums(ConcurrentDictionary results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> documentedEnums) + private static int GetCommonPrefixLength(ReadOnlySpan first, ReadOnlySpan second) + { + int count = 0; + int minLength = Math.Min(first.Length, second.Length); + for (int i = 0; i < minLength; i++) + { + if (first[i] == second[i]) + { + count++; + } + else + { + break; + } + } + + return count; + } + + private int AnalyzeEnums(ConcurrentDictionary results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> documentedEnums) { var uniqueEnums = new Dictionary>(); var constantsDocs = new Dictionary>(); @@ -91,16 +119,16 @@ private static int AnalyzeEnums(ConcurrentDictionary results list.Add(item.Key); - foreach (KeyValuePair enumValue in item.Value.MemberNamesAndDocs) + foreach (KeyValuePair enumValue in item.Value.Members) { - if (enumValue.Value is object) + if (enumValue.Value.Doc is object) { if (!constantsDocs.TryGetValue(enumValue.Key, out List<(string MethodName, string HelpLink, string Doc)>? values)) { constantsDocs.Add(enumValue.Key, values = new()); } - values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value)); + values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value.Doc)); } } } @@ -141,6 +169,102 @@ private static int AnalyzeEnums(ConcurrentDictionary results } } + if (this.EmitEnums) + { + using var metadataStream = File.OpenRead(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "Windows.Win32.winmd")); + using var peReader = new PEReader(metadataStream); + var mr = peReader.GetMetadataReader(); + + var apiClassHandles = new HashSet( + from typeDefHandle in mr.TypeDefinitions + let typeDef = mr.GetTypeDefinition(typeDefHandle) + where mr.StringComparer.Equals(typeDef.Name, "Apis") + select typeDefHandle); + + string enumDirectory = Path.GetDirectoryName(this.outputPath) ?? throw new InvalidOperationException("Unable to determine where to write enums."); + using var enumRewrite = new StreamWriter(File.OpenWrite(Path.Combine(enumDirectory, "remap.rsp"))); + var enumsByNamespace = new Dictionary>(); + int anonymousEnumCounter = 0; + foreach (KeyValuePair> item in uniqueEnums) + { + string? enumName = null; + if (item.Value.Count == 1) + { + var oneValue = item.Value[0]; + if (oneValue.ParameterName.Contains("flags", StringComparison.OrdinalIgnoreCase)) + { + // Only appears in one method, on a parameter named something like "flags". + enumName = $"{oneValue.MethodName}Flags"; + } + else + { + enumName = $"{oneValue.MethodName}_{oneValue.ParameterName}Flags"; + } + } + else + { + string firstName = item.Key.Members.Keys.First(); + int commonPrefixLength = firstName.Length; + foreach (string key in item.Key.Members.Keys) + { + commonPrefixLength = Math.Min(commonPrefixLength, GetCommonPrefixLength(key, firstName)); + } + + if (commonPrefixLength > 1) + { + int last_ = firstName.LastIndexOf('_', commonPrefixLength - 1); + if (last_ != -1 && last_ != commonPrefixLength - 1) + { + // Trim down to last underscore + commonPrefixLength = last_; + } + + if (commonPrefixLength > 1 && firstName[commonPrefixLength - 1] == '_') + { + // The enum values share a common prefix suitable to imply a name for the enum. + enumName = firstName.Substring(0, commonPrefixLength - 1); + } + } + + if (enumName is null) + { + enumName = $"AnonymousEnum{++anonymousEnumCounter}"; + } + } + + var enumWithNamespace = item.Key.Emit(enumName, mr, apiClassHandles); + if (enumWithNamespace is object) + { + if (!enumsByNamespace.TryGetValue(enumWithNamespace.Value.Namespace, out var list)) + { + enumsByNamespace.Add(enumWithNamespace.Value.Namespace, list = new()); + } + + list.Add(enumWithNamespace.Value.Enum); + + foreach (var tuple in item.Value) + { + enumRewrite.WriteLine($"{tuple.MethodName}:{tuple.ParameterName}={enumName}"); + } + } + } + + foreach (var e in enumsByNamespace) + { + var compilationUnit = SyntaxFactory.CompilationUnit() + .AddUsings(SyntaxFactory.UsingDirective(SyntaxFactory.IdentifierName("System"))) + .AddMembers( + SyntaxFactory.NamespaceDeclaration(SyntaxFactory.ParseName(e.Key)) + .AddMembers(e.Value.ToArray())) + .NormalizeWhitespace(); + + string simpleNamespace = e.Key.Substring("Windows.Win32.".Length); + File.WriteAllText( + Path.Combine(enumDirectory, $"{simpleNamespace}.manual.cs"), + compilationUnit.ToFullString()); + } + } + return constantsDocs.Count; } @@ -186,7 +310,7 @@ where result is not null } Console.WriteLine("Analyzing and naming enums and collecting docs on their members..."); - int constantsCount = AnalyzeEnums(results, documentedEnums); + int constantsCount = this.AnalyzeEnums(results, documentedEnums); Console.WriteLine($"Found docs for {constantsCount} constants."); Console.WriteLine("Writing results to \"{0}\"", this.outputPath); @@ -302,15 +426,16 @@ void ParseTextSection(out YamlScalarNode node) docBuilder.Clear(); } - IReadOnlyDictionary ParseEnumTable() + IReadOnlyDictionary ParseEnumTable() { - var enums = new Dictionary(); + var enums = new Dictionary(); int state = 0; const int StateReadingHeader = 0; const int StateReadingName = 1; - const int StateLookingForDocColumn = 2; + const int StateLookingForDetail = 2; const int StateReadingDocColumn = 3; string? enumName = null; + ulong? enumValue = null; var docsBuilder = new StringBuilder(); while ((line = mdFileReader.ReadLine()) is object) { @@ -336,14 +461,32 @@ void ParseTextSection(out YamlScalarNode node) if (m.Success) { enumName = m.Groups[1].Value; - state = StateLookingForDocColumn; + if (enumName == "0") + { + enumName = "None"; + enumValue = 0; + } + + state = StateLookingForDetail; } break; - case 2: + case StateLookingForDetail: // Looking for an enum row's doc column. - if (line.StartsWith("", StringComparison.OrdinalIgnoreCase)) { @@ -366,10 +510,11 @@ void ParseTextSection(out YamlScalarNode node) // Some docs are invalid in documenting the same enum multiple times. if (!enums.ContainsKey(enumName!)) { - enums.Add(enumName!, docsBuilder.ToString().Trim()); + enums.Add(enumName!, (enumValue, docsBuilder.ToString().Trim())); } enumName = null; + enumValue = null; docsBuilder.Clear(); break; } @@ -400,16 +545,26 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = { if (line == "") { - IReadOnlyDictionary enumNamesAndDocs = ParseEnumTable(); - enumsByParameter ??= new Dictionary(); - enumsByParameter.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); + IReadOnlyDictionary enumNamesAndDocs = ParseEnumTable(); + if (enumNamesAndDocs.Count > 0) + { + enumsByParameter ??= new Dictionary(); + if (!enumsByParameter.ContainsKey(sectionName)) + { + enumsByParameter.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); + } + } + lookForEnums = false; } } else { foundEnum = line.Contains("of the following values", StringComparison.OrdinalIgnoreCase); - foundEnumIsFlags = line.Contains("combination of", StringComparison.OrdinalIgnoreCase); + foundEnumIsFlags = line.Contains("combination of", StringComparison.OrdinalIgnoreCase) + || line.Contains("zero or more of", StringComparison.OrdinalIgnoreCase) + || line.Contains("one or both of", StringComparison.OrdinalIgnoreCase) + || line.Contains("one or more of", StringComparison.OrdinalIgnoreCase); } } @@ -439,7 +594,7 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = } else if (FieldHeaderPattern.Match(line) is Match { Success: true } fieldMatch) { - ParseSection(fieldMatch, fieldsMap); + ParseSection(fieldMatch, fieldsMap, lookForEnums: true); } else if (RemarksHeaderPattern.Match(line) is Match { Success: true } remarksMatch) { diff --git a/src/ScrapeDocs/ScrapeDocs.csproj b/src/ScrapeDocs/ScrapeDocs.csproj index 5bd25a75..df0672f7 100644 --- a/src/ScrapeDocs/ScrapeDocs.csproj +++ b/src/ScrapeDocs/ScrapeDocs.csproj @@ -7,6 +7,18 @@ + + + + + + PreserveNewest + + + + + + From ecc8a665d8f1e7fa76d85f97e6f0bbd9de160a35 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Mar 2021 12:35:45 -0700 Subject: [PATCH 2/2] Generate json with enums instead of C# --- src/Microsoft.Windows.CsWin32/Generator.cs | 58 ++++- .../GeneratorUtilities.cs | 93 ------- src/ScrapeDocs/DocEnum.cs | 115 ++++----- src/ScrapeDocs/Program.cs | 236 ++++++++---------- src/ScrapeDocs/ScrapeDocs.csproj | 12 - 5 files changed, 202 insertions(+), 312 deletions(-) delete mode 100644 src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs diff --git a/src/Microsoft.Windows.CsWin32/Generator.cs b/src/Microsoft.Windows.CsWin32/Generator.cs index 8080a6ee..0415e48e 100644 --- a/src/Microsoft.Windows.CsWin32/Generator.cs +++ b/src/Microsoft.Windows.CsWin32/Generator.cs @@ -3426,8 +3426,62 @@ private MarshalAsAttribute ToMarshalAsAttribute(BlobHandle blobHandle) return ma; } - private ExpressionSyntax ToExpressionSyntax(Constant constant) => GeneratorUtilities.ToExpressionSyntax(this.mr, constant); + private ExpressionSyntax ToExpressionSyntax(Constant constant) + { + var blobReader = this.mr.GetBlobReader(constant.Value); + return constant.TypeCode switch + { + ConstantTypeCode.Boolean => blobReader.ReadBoolean() ? LiteralExpression(SyntaxKind.TrueLiteralExpression) : LiteralExpression(SyntaxKind.FalseLiteralExpression), + ConstantTypeCode.Char => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadChar())), + ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadSByte())), + ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadByte())), + ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt16())), + ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt16())), + ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt32())), + ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt32())), + ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt64())), + ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt64())), + ConstantTypeCode.Single => FloatExpression(blobReader.ReadSingle()), + ConstantTypeCode.Double => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadDouble())), + ConstantTypeCode.String => blobReader.ReadConstant(constant.TypeCode) is string value ? LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(value)) : LiteralExpression(SyntaxKind.NullLiteralExpression), + ConstantTypeCode.NullReference => LiteralExpression(SyntaxKind.NullLiteralExpression), + _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), + }; - private ExpressionSyntax ToHexExpressionSyntax(Constant constant) => GeneratorUtilities.ToHexExpressionSyntax(this.mr, constant); + static ExpressionSyntax FloatExpression(float value) + { + return + float.IsPositiveInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.PositiveInfinity))) : + float.IsNegativeInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NegativeInfinity))) : + float.IsNaN(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NaN))) : + LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(value)); + } + } + + private ExpressionSyntax ToHexExpressionSyntax(Constant constant) + { + var blobReader = this.mr.GetBlobReader(constant.Value); + var blobReader2 = this.mr.GetBlobReader(constant.Value); + return constant.TypeCode switch + { + ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadSByte()), blobReader2.ReadSByte())), + ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadByte()), blobReader2.ReadByte())), + ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt16()), blobReader2.ReadInt16())), + ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt16()), blobReader2.ReadUInt16())), + ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt32()), blobReader2.ReadInt32())), + ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt32()), blobReader2.ReadUInt32())), + ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt64()), blobReader2.ReadInt64())), + ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt64()), blobReader2.ReadUInt64())), + _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), + }; + + unsafe string ToHex(T value) + where T : unmanaged + { + int fullHexLength = sizeof(T) * 2; + string hex = string.Format(CultureInfo.InvariantCulture, "0x{0:X" + fullHexLength + "}", value); + return hex; + } + } } } diff --git a/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs b/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs deleted file mode 100644 index 4091ecf7..00000000 --- a/src/Microsoft.Windows.CsWin32/GeneratorUtilities.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Globalization; -using System.Reflection.Metadata; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using static Microsoft.CodeAnalysis.CSharp.SyntaxFactory; - -internal static class GeneratorUtilities -{ - internal static PredefinedTypeSyntax ToTypeOfConstant(MetadataReader mr, Constant constant) - { - var blobReader = mr.GetBlobReader(constant.Value); - SyntaxKind keyword = constant.TypeCode switch - { - ConstantTypeCode.Boolean => SyntaxKind.BoolKeyword, - ConstantTypeCode.Char => SyntaxKind.CharKeyword, - ConstantTypeCode.SByte => SyntaxKind.SByteKeyword, - ConstantTypeCode.Byte => SyntaxKind.ByteKeyword, - ConstantTypeCode.Int16 => SyntaxKind.ShortKeyword, - ConstantTypeCode.UInt16 => SyntaxKind.UShortKeyword, - ConstantTypeCode.Int32 => SyntaxKind.IntKeyword, - ConstantTypeCode.UInt32 => SyntaxKind.UIntKeyword, - ConstantTypeCode.Int64 => SyntaxKind.LongKeyword, - ConstantTypeCode.UInt64 => SyntaxKind.ULongKeyword, - ConstantTypeCode.Single => SyntaxKind.FloatKeyword, - ConstantTypeCode.Double => SyntaxKind.DoubleKeyword, - ConstantTypeCode.String => SyntaxKind.StringKeyword, - _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), - }; - return PredefinedType(Token(keyword)); - } - - internal static ExpressionSyntax ToExpressionSyntax(MetadataReader mr, Constant constant) - { - var blobReader = mr.GetBlobReader(constant.Value); - return constant.TypeCode switch - { - ConstantTypeCode.Boolean => blobReader.ReadBoolean() ? LiteralExpression(SyntaxKind.TrueLiteralExpression) : LiteralExpression(SyntaxKind.FalseLiteralExpression), - ConstantTypeCode.Char => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadChar())), - ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadSByte())), - ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadByte())), - ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt16())), - ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt16())), - ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt32())), - ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt32())), - ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadInt64())), - ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadUInt64())), - ConstantTypeCode.Single => FloatExpression(blobReader.ReadSingle()), - ConstantTypeCode.Double => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(blobReader.ReadDouble())), - ConstantTypeCode.String => blobReader.ReadConstant(constant.TypeCode) is string value ? LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(value)) : LiteralExpression(SyntaxKind.NullLiteralExpression), - ConstantTypeCode.NullReference => LiteralExpression(SyntaxKind.NullLiteralExpression), - _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), - }; - - static ExpressionSyntax FloatExpression(float value) - { - return - float.IsPositiveInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.PositiveInfinity))) : - float.IsNegativeInfinity(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NegativeInfinity))) : - float.IsNaN(value) ? MemberAccessExpression(SyntaxKind.SimpleMemberAccessExpression, PredefinedType(Token(SyntaxKind.FloatKeyword)), IdentifierName(nameof(float.NaN))) : - LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(value)); - } - } - - internal static ExpressionSyntax ToHexExpressionSyntax(MetadataReader mr, Constant constant) - { - var blobReader = mr.GetBlobReader(constant.Value); - var blobReader2 = mr.GetBlobReader(constant.Value); - return constant.TypeCode switch - { - ConstantTypeCode.SByte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadSByte()), blobReader2.ReadSByte())), - ConstantTypeCode.Byte => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadByte()), blobReader2.ReadByte())), - ConstantTypeCode.Int16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt16()), blobReader2.ReadInt16())), - ConstantTypeCode.UInt16 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt16()), blobReader2.ReadUInt16())), - ConstantTypeCode.Int32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt32()), blobReader2.ReadInt32())), - ConstantTypeCode.UInt32 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt32()), blobReader2.ReadUInt32())), - ConstantTypeCode.Int64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadInt64()), blobReader2.ReadInt64())), - ConstantTypeCode.UInt64 => LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(ToHex(blobReader.ReadUInt64()), blobReader2.ReadUInt64())), - _ => throw new NotSupportedException("ConstantTypeCode not supported: " + constant.TypeCode), - }; - - unsafe string ToHex(T value) - where T : unmanaged - { - int fullHexLength = sizeof(T) * 2; - string hex = string.Format(CultureInfo.InvariantCulture, "0x{0:X" + fullHexLength + "}", value); - return hex; - } - } -} diff --git a/src/ScrapeDocs/DocEnum.cs b/src/ScrapeDocs/DocEnum.cs index 574e5f86..83946966 100644 --- a/src/ScrapeDocs/DocEnum.cs +++ b/src/ScrapeDocs/DocEnum.cs @@ -3,18 +3,12 @@ namespace ScrapeDocs { + using System; using System.Collections.Generic; using System.Linq; - using System.Reflection.Metadata; - using Microsoft.CodeAnalysis.CSharp; - using Microsoft.CodeAnalysis.CSharp.Syntax; - using static GeneratorUtilities; - using static Microsoft.CodeAnalysis.CSharp.SyntaxFactory; internal class DocEnum { - private static readonly AttributeListSyntax FlagsAttributeList = AttributeList().AddAttributes(Attribute(IdentifierName("Flags"))); - internal DocEnum(bool isFlags, IReadOnlyDictionary members) { this.IsFlags = isFlags; @@ -75,83 +69,68 @@ public bool Equals(DocEnum? other) return true; } - internal (string Namespace, EnumDeclarationSyntax Enum)? Emit(string name, MetadataReader mr, HashSet apiClassHandles) + internal string? GetRecommendedName(List<(string MethodName, string ParameterName, string HelpLink, bool IsMethod)> uses) { - if (this.Members.Count == 2 && this.Members.ContainsKey("TRUE") && this.Members.ContainsKey("FALSE")) - { - return null; - } - - PredefinedTypeSyntax? baseType = null; - string? ns = null; - - // Look up values for each constant. - var values = new Dictionary(); - foreach (var item in this.Members) + string? enumName = null; + if (uses.Count == 1) { - bool found = false; - foreach (FieldDefinitionHandle handle in mr.FieldDefinitions) + var oneValue = uses[0]; + if (oneValue.ParameterName.Contains("flags", StringComparison.OrdinalIgnoreCase)) { - FieldDefinition fieldDef = mr.GetFieldDefinition(handle); - if (apiClassHandles.Contains(fieldDef.GetDeclaringType()) && mr.StringComparer.Equals(fieldDef.Name, item.Key)) - { - found = true; - Constant constant = mr.GetConstant(fieldDef.GetDefaultValue()); - values.Add(item.Key, this.IsFlags ? ToHexExpressionSyntax(mr, constant) : ToExpressionSyntax(mr, constant)); - baseType ??= ToTypeOfConstant(mr, constant); - ns ??= mr.GetString(mr.GetTypeDefinition(fieldDef.GetDeclaringType()).Namespace); - break; - } + // Only appears in one method, on a parameter named something like "flags". + enumName = $"{oneValue.MethodName}Flags"; } - - if (!found) + else { - // We couldn't find all the constants required. - return null; + enumName = $"{oneValue.MethodName}_{oneValue.ParameterName}Flags"; } } - - if (baseType is null || ns is null) - { - // We don't know all the values. - return null; - } - - // Strip the method's declaring interface from the enum name, where applicable. - NameSyntax enumNameSyntax = ParseName(name); - if (enumNameSyntax is QualifiedNameSyntax qname) + else { - enumNameSyntax = qname.Right; - } - - EnumDeclarationSyntax enumDecl = EnumDeclaration(Identifier(enumNameSyntax.ToString())) - .AddModifiers(Token(SyntaxKind.PublicKeyword)) - .AddMembers(this.Members - .Select(kv => EnumMemberDeclaration(Identifier(kv.Key)).WithEqualsValue(EqualsValueClause(values[kv.Key]))).ToArray()); + string firstName = this.Members.Keys.First(); + int commonPrefixLength = firstName.Length; + foreach (string key in this.Members.Keys) + { + commonPrefixLength = Math.Min(commonPrefixLength, GetCommonPrefixLength(key, firstName)); + } - if (this.IsFlags) - { - // For flags enums, prefer typing as unsigned integers. - baseType = PredefinedType(Token(baseType.Keyword.Kind() switch + if (commonPrefixLength > 1) { - SyntaxKind.ShortKeyword => SyntaxKind.UShortKeyword, - SyntaxKind.IntKeyword => SyntaxKind.UIntKeyword, - SyntaxKind.LongKeyword => SyntaxKind.ULongKeyword, - _ => baseType.Keyword.Kind(), - })); - } + int last_ = firstName.LastIndexOf('_', commonPrefixLength - 1); + if (last_ != -1 && last_ != commonPrefixLength - 1) + { + // Trim down to last underscore + commonPrefixLength = last_; + } - if (baseType is not PredefinedTypeSyntax { Keyword: { RawKind: (int)SyntaxKind.IntKeyword } }) - { - enumDecl = enumDecl.AddBaseListTypes(SimpleBaseType(baseType)); + if (commonPrefixLength > 1 && firstName[commonPrefixLength - 1] == '_') + { + // The enum values share a common prefix suitable to imply a name for the enum. + enumName = firstName.Substring(0, commonPrefixLength - 1); + } + } } - if (this.IsFlags) + return enumName; + } + + private static int GetCommonPrefixLength(ReadOnlySpan first, ReadOnlySpan second) + { + int count = 0; + int minLength = Math.Min(first.Length, second.Length); + for (int i = 0; i < minLength; i++) { - enumDecl = enumDecl.AddAttributeLists(FlagsAttributeList); + if (first[i] == second[i]) + { + count++; + } + else + { + break; + } } - return (ns, enumDecl); + return count; } } } diff --git a/src/ScrapeDocs/Program.cs b/src/ScrapeDocs/Program.cs index 6646ef8b..5fce65f5 100644 --- a/src/ScrapeDocs/Program.cs +++ b/src/ScrapeDocs/Program.cs @@ -6,21 +6,16 @@ namespace ScrapeDocs using System; using System.Collections.Concurrent; using System.Collections.Generic; - using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Linq; using System.Reflection; - using System.Reflection.Metadata; - using System.Reflection.PortableExecutable; using System.Text; + using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; - using Microsoft.CodeAnalysis; - using Microsoft.CodeAnalysis.CSharp; - using Microsoft.CodeAnalysis.CSharp.Syntax; using YamlDotNet.RepresentationModel; /// @@ -87,52 +82,43 @@ private static void Expect(string? expected, string? actual) } } - private static int GetCommonPrefixLength(ReadOnlySpan first, ReadOnlySpan second) - { - int count = 0; - int minLength = Math.Min(first.Length, second.Length); - for (int i = 0; i < minLength; i++) - { - if (first[i] == second[i]) - { - count++; - } - else - { - break; - } - } + // Skip the NULL constant due to https://github.com/aaubry/YamlDotNet/issues/591. + private static bool IsYamlProblematicKey(string key) => string.Equals(key, "null", StringComparison.OrdinalIgnoreCase); - return count; - } - - private int AnalyzeEnums(ConcurrentDictionary results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> documentedEnums) + private int AnalyzeEnums(ConcurrentDictionary results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> parameterEnums, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> fieldEnums) { - var uniqueEnums = new Dictionary>(); - var constantsDocs = new Dictionary>(); - foreach (var item in documentedEnums) + var uniqueEnums = new Dictionary>(); + var constantsDocs = new Dictionary>(); + + void Collect(ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> enums, bool isMethod) { - if (!uniqueEnums.TryGetValue(item.Value, out List<(string MethodName, string ParameterName, string HelpLink)>? list)) + foreach (var item in enums) { - uniqueEnums.Add(item.Value, list = new()); - } + if (!uniqueEnums.TryGetValue(item.Value, out List<(string MethodName, string ParameterName, string HelpLink, bool IsMethod)>? list)) + { + uniqueEnums.Add(item.Value, list = new()); + } - list.Add(item.Key); + list.Add((item.Key.MethodName, item.Key.ParameterName, item.Key.HelpLink, isMethod)); - foreach (KeyValuePair enumValue in item.Value.Members) - { - if (enumValue.Value.Doc is object) + foreach (KeyValuePair enumValue in item.Value.Members) { - if (!constantsDocs.TryGetValue(enumValue.Key, out List<(string MethodName, string HelpLink, string Doc)>? values)) + if (enumValue.Value.Doc is object) { - constantsDocs.Add(enumValue.Key, values = new()); - } + if (!constantsDocs.TryGetValue(enumValue.Key, out List<(string MethodName, string HelpLink, string Doc)>? values)) + { + constantsDocs.Add(enumValue.Key, values = new()); + } - values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value.Doc)); + values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value.Doc)); + } } } } + Collect(parameterEnums, isMethod: true); + Collect(fieldEnums, isMethod: false); + foreach (var item in constantsDocs) { string doc = item.Value[0].Doc; @@ -151,7 +137,7 @@ private int AnalyzeEnums(ConcurrentDictionary results, Concu var docNode = new YamlMappingNode(); if (differenceDetected) { - doc = "Documentation varies per use. Refer to each: " + string.Join(", ", item.Value.Select(v => @$"{v.MethodName}")) + "."; + doc = "Documentation varies per use. Refer to each: " + string.Join(", ", item.Value.Select(v => @$"{v.MethodOrStructName}")) + "."; } else { @@ -161,9 +147,7 @@ private int AnalyzeEnums(ConcurrentDictionary results, Concu docNode.Add("Description", doc); - // Skip the NULL constant due to https://github.com/aaubry/YamlDotNet/issues/591. - // Besides--documenting null probably isn't necessary. - if (item.Key != "NULL") + if (!IsYamlProblematicKey(item.Key)) { results.TryAdd(new YamlScalarNode(item.Key), docNode); } @@ -171,98 +155,65 @@ private int AnalyzeEnums(ConcurrentDictionary results, Concu if (this.EmitEnums) { - using var metadataStream = File.OpenRead(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "Windows.Win32.winmd")); - using var peReader = new PEReader(metadataStream); - var mr = peReader.GetMetadataReader(); - - var apiClassHandles = new HashSet( - from typeDefHandle in mr.TypeDefinitions - let typeDef = mr.GetTypeDefinition(typeDefHandle) - where mr.StringComparer.Equals(typeDef.Name, "Apis") - select typeDefHandle); - string enumDirectory = Path.GetDirectoryName(this.outputPath) ?? throw new InvalidOperationException("Unable to determine where to write enums."); - using var enumRewrite = new StreamWriter(File.OpenWrite(Path.Combine(enumDirectory, "remap.rsp"))); - var enumsByNamespace = new Dictionary>(); - int anonymousEnumCounter = 0; - foreach (KeyValuePair> item in uniqueEnums) + Directory.CreateDirectory(enumDirectory); + using var enumsJsonStream = File.OpenWrite(Path.Combine(enumDirectory, "enums.json")); + using var writer = new Utf8JsonWriter(enumsJsonStream, new JsonWriterOptions { Indented = true }); + writer.WriteStartArray(); + + foreach (KeyValuePair> item in uniqueEnums) { - string? enumName = null; - if (item.Value.Count == 1) + writer.WriteStartObject(); + + if (item.Key.GetRecommendedName(item.Value) is string enumName) { - var oneValue = item.Value[0]; - if (oneValue.ParameterName.Contains("flags", StringComparison.OrdinalIgnoreCase)) - { - // Only appears in one method, on a parameter named something like "flags". - enumName = $"{oneValue.MethodName}Flags"; - } - else - { - enumName = $"{oneValue.MethodName}_{oneValue.ParameterName}Flags"; - } + writer.WriteString("name", enumName); } - else - { - string firstName = item.Key.Members.Keys.First(); - int commonPrefixLength = firstName.Length; - foreach (string key in item.Key.Members.Keys) - { - commonPrefixLength = Math.Min(commonPrefixLength, GetCommonPrefixLength(key, firstName)); - } - - if (commonPrefixLength > 1) - { - int last_ = firstName.LastIndexOf('_', commonPrefixLength - 1); - if (last_ != -1 && last_ != commonPrefixLength - 1) - { - // Trim down to last underscore - commonPrefixLength = last_; - } - if (commonPrefixLength > 1 && firstName[commonPrefixLength - 1] == '_') - { - // The enum values share a common prefix suitable to imply a name for the enum. - enumName = firstName.Substring(0, commonPrefixLength - 1); - } - } + writer.WriteBoolean("flags", item.Key.IsFlags); - if (enumName is null) + writer.WritePropertyName("members"); + writer.WriteStartArray(); + foreach (var member in item.Key.Members) + { + writer.WriteStartObject(); + writer.WriteString("name", member.Key); + if (member.Value.Value is ulong value) { - enumName = $"AnonymousEnum{++anonymousEnumCounter}"; + writer.WriteString("value", value.ToString(CultureInfo.InvariantCulture)); } + + writer.WriteEndObject(); } - var enumWithNamespace = item.Key.Emit(enumName, mr, apiClassHandles); - if (enumWithNamespace is object) + writer.WriteEndArray(); + + writer.WritePropertyName("uses"); + writer.WriteStartArray(); + foreach (var uses in item.Value) { - if (!enumsByNamespace.TryGetValue(enumWithNamespace.Value.Namespace, out var list)) - { - enumsByNamespace.Add(enumWithNamespace.Value.Namespace, list = new()); - } + writer.WriteStartObject(); - list.Add(enumWithNamespace.Value.Enum); + int periodIndex = uses.MethodName.IndexOf('.', StringComparison.Ordinal); + string? iface = periodIndex >= 0 ? uses.MethodName.Substring(0, periodIndex) : null; + string name = periodIndex >= 0 ? uses.MethodName.Substring(periodIndex + 1) : uses.MethodName; - foreach (var tuple in item.Value) + if (iface is string) { - enumRewrite.WriteLine($"{tuple.MethodName}:{tuple.ParameterName}={enumName}"); + writer.WriteString("interface", iface); } + + writer.WriteString(uses.IsMethod ? "method" : "struct", name); + writer.WriteString(uses.IsMethod ? "parameter" : "field", uses.ParameterName); + + writer.WriteEndObject(); } - } - foreach (var e in enumsByNamespace) - { - var compilationUnit = SyntaxFactory.CompilationUnit() - .AddUsings(SyntaxFactory.UsingDirective(SyntaxFactory.IdentifierName("System"))) - .AddMembers( - SyntaxFactory.NamespaceDeclaration(SyntaxFactory.ParseName(e.Key)) - .AddMembers(e.Value.ToArray())) - .NormalizeWhitespace(); - - string simpleNamespace = e.Key.Substring("Windows.Win32.".Length); - File.WriteAllText( - Path.Combine(enumDirectory, $"{simpleNamespace}.manual.cs"), - compilationUnit.ToFullString()); + writer.WriteEndArray(); + writer.WriteEndObject(); } + + writer.WriteEndArray(); } return constantsDocs.Count; @@ -272,7 +223,7 @@ private void Worker(CancellationToken cancellationToken) { Console.WriteLine("Enumerating documents to be parsed..."); string[] paths = Directory.GetFiles(this.contentBasePath, "??-*-*.md", SearchOption.AllDirectories) - ////.Where(p => p.Contains(@"ext\sdk-api\sdk-api-src\content\winuser\nf-winuser-setwindowpos.md")).ToArray() + ////.Where(p => p.Contains(@"DNS_RECORDA", StringComparison.OrdinalIgnoreCase)).ToArray() ; Console.WriteLine("Parsing documents..."); @@ -280,23 +231,30 @@ private void Worker(CancellationToken cancellationToken) var parsedNodes = from path in paths.AsParallel() let result = this.ParseDocFile(path) where result is not null - select (Path: path, result.Value.ApiName, result.Value.YamlNode, result.Value.EnumsByParameter); + select (Path: path, result.Value.ApiName, result.Value.YamlNode, result.Value.EnumsByParameter, result.Value.EnumsByField); var results = new ConcurrentDictionary(); - var documentedEnums = new ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum>(); + var parameterEnums = new ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum>(); + var fieldEnums = new ConcurrentDictionary<(string StructName, string FieldName, string HelpLink), DocEnum>(); if (Debugger.IsAttached) { parsedNodes = parsedNodes.WithDegreeOfParallelism(1); // improve debuggability } parsedNodes - .WithCancellation<(string Path, string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter)>(cancellationToken) + .WithCancellation<(string Path, string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter, IReadOnlyDictionary EnumsByField)>(cancellationToken) .ForAll(result => { results.TryAdd(new YamlScalarNode(result.ApiName), result.YamlNode); foreach (var e in result.EnumsByParameter) { string helpLink = ((YamlScalarNode)result.YamlNode["HelpLink"]).Value!; - documentedEnums.TryAdd((result.ApiName, e.Key, helpLink), e.Value); + parameterEnums.TryAdd((result.ApiName, e.Key, helpLink), e.Value); + } + + foreach (var e in result.EnumsByField) + { + string helpLink = ((YamlScalarNode)result.YamlNode["HelpLink"]).Value!; + fieldEnums.TryAdd((result.ApiName, e.Key, helpLink), e.Value); } }); if (paths.Length == 0) @@ -306,11 +264,11 @@ where result is not null else { Console.WriteLine("Parsed {2} documents in {0} ({1} per document)", timer.Elapsed, timer.Elapsed / paths.Length, paths.Length); - Console.WriteLine($"Found {documentedEnums.Count} enums."); + Console.WriteLine($"Found {parameterEnums.Count + fieldEnums.Count} enums."); } Console.WriteLine("Analyzing and naming enums and collecting docs on their members..."); - int constantsCount = this.AnalyzeEnums(results, documentedEnums); + int constantsCount = this.AnalyzeEnums(results, parameterEnums, fieldEnums); Console.WriteLine($"Found docs for {constantsCount} constants."); Console.WriteLine("Writing results to \"{0}\"", this.outputPath); @@ -322,11 +280,12 @@ where result is not null yamlStream.Save(yamlWriter); } - private (string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter)? ParseDocFile(string filePath) + private (string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter, IReadOnlyDictionary EnumsByField)? ParseDocFile(string filePath) { try { - IDictionary? enumsByParameter = null; + var enumsByParameter = new Dictionary(); + var enumsByField = new Dictionary(); var yaml = new YamlStream(); using StreamReader mdFileReader = File.OpenText(filePath); using var markdownToYamlReader = new YamlSectionReader(mdFileReader); @@ -527,7 +486,7 @@ void ParseTextSection(out YamlScalarNode node) return enums; } - void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = false) + void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForParameterEnums = false, bool lookForFieldEnums = false) { string sectionName = match.Groups[1].Value; bool foundEnum = false; @@ -539,7 +498,7 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = break; } - if (lookForEnums) + if (lookForParameterEnums || lookForFieldEnums) { if (foundEnum) { @@ -548,14 +507,15 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = IReadOnlyDictionary enumNamesAndDocs = ParseEnumTable(); if (enumNamesAndDocs.Count > 0) { - enumsByParameter ??= new Dictionary(); - if (!enumsByParameter.ContainsKey(sectionName)) + var enums = lookForParameterEnums ? enumsByParameter : enumsByField; + if (!enums.ContainsKey(sectionName)) { - enumsByParameter.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); + enums.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); } } - lookForEnums = false; + lookForParameterEnums = false; + lookForFieldEnums = false; } } else @@ -577,7 +537,10 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = try { - receivingMap.Add(sectionName, docBuilder.ToString().Trim()); + if (!IsYamlProblematicKey(sectionName)) + { + receivingMap.Add(sectionName, docBuilder.ToString().Trim()); + } } catch (ArgumentException) { @@ -590,11 +553,11 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = { if (ParameterHeaderPattern.Match(line) is Match { Success: true } parameterMatch) { - ParseSection(parameterMatch, parametersMap, lookForEnums: true); + ParseSection(parameterMatch, parametersMap, lookForParameterEnums: true); } else if (FieldHeaderPattern.Match(line) is Match { Success: true } fieldMatch) { - ParseSection(fieldMatch, fieldsMap, lookForEnums: true); + ParseSection(fieldMatch, fieldsMap, lookForFieldEnums: true); } else if (RemarksHeaderPattern.Match(line) is Match { Success: true } remarksMatch) { @@ -652,8 +615,7 @@ void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = } } - enumsByParameter ??= ImmutableDictionary.Empty; - return (properName, methodNode, (IReadOnlyDictionary)enumsByParameter); + return (properName, methodNode, enumsByParameter, enumsByField); } catch (Exception ex) { diff --git a/src/ScrapeDocs/ScrapeDocs.csproj b/src/ScrapeDocs/ScrapeDocs.csproj index df0672f7..5bd25a75 100644 --- a/src/ScrapeDocs/ScrapeDocs.csproj +++ b/src/ScrapeDocs/ScrapeDocs.csproj @@ -7,18 +7,6 @@ - - - - - - PreserveNewest - - - - - -