From 90698c91f979afe698bb1e0ce6408d79e11f2cae Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 2 Mar 2021 10:59:10 -0700 Subject: [PATCH] Add documentation about constants Closes #22 --- src/Microsoft.Windows.CsWin32/Generator.cs | 29 +- src/ScrapeDocs/DocEnum.cs | 70 +++ src/ScrapeDocs/Program.cs | 495 +++++++++++++++------ 3 files changed, 436 insertions(+), 158 deletions(-) create mode 100644 src/ScrapeDocs/DocEnum.cs diff --git a/src/Microsoft.Windows.CsWin32/Generator.cs b/src/Microsoft.Windows.CsWin32/Generator.cs index 40fae608..0415e48e 100644 --- a/src/Microsoft.Windows.CsWin32/Generator.cs +++ b/src/Microsoft.Windows.CsWin32/Generator.cs @@ -905,7 +905,9 @@ internal void GenerateConstant(FieldDefinitionHandle fieldDefHandle) return; } - this.fieldsToSyntax.Add(fieldDefHandle, this.CreateField(fieldDefHandle)); + FieldDeclarationSyntax constantDeclaration = this.CreateField(fieldDefHandle); + constantDeclaration = AddApiDocumentation(constantDeclaration.Declaration.Variables[0].Identifier.ValueText, constantDeclaration); + this.fieldsToSyntax.Add(fieldDefHandle, constantDeclaration); } internal TypeSyntax? GenerateSafeHandle(string releaseMethod) @@ -1208,19 +1210,22 @@ private static T AddApiDocumentation(string api, T memberDeclaration) docCommentsBuilder.AppendLine(""); } - docCommentsBuilder.Append($"/// "); - if (docs.Remarks is object) + if (docs.Remarks is object || docs.HelpLink is object) { - EmitDoc(docs.Remarks, docCommentsBuilder, docs, string.Empty); - } - else - { - docCommentsBuilder.AppendLine(); - docCommentsBuilder.AppendLine($@"/// Learn more about this API from docs.microsoft.com."); - docCommentsBuilder.Append("/// "); - } + docCommentsBuilder.Append($"/// "); + if (docs.Remarks is object) + { + EmitDoc(docs.Remarks, docCommentsBuilder, docs, string.Empty); + } + else if (docs.HelpLink is object) + { + docCommentsBuilder.AppendLine(); + docCommentsBuilder.AppendLine($@"/// Learn more about this API from docs.microsoft.com."); + docCommentsBuilder.Append("/// "); + } - docCommentsBuilder.AppendLine($""); + docCommentsBuilder.AppendLine($""); + } memberDeclaration = memberDeclaration.WithLeadingTrivia( ParseLeadingTrivia(docCommentsBuilder.ToString())); diff --git a/src/ScrapeDocs/DocEnum.cs b/src/ScrapeDocs/DocEnum.cs new file mode 100644 index 00000000..d28e7988 --- /dev/null +++ b/src/ScrapeDocs/DocEnum.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace ScrapeDocs +{ + using System.Collections.Generic; + + internal class DocEnum + { + internal DocEnum(bool isFlags, IReadOnlyDictionary memberNamesAndDocs) + { + this.IsFlags = isFlags; + this.MemberNamesAndDocs = memberNamesAndDocs; + } + + internal bool IsFlags { get; } + + internal IReadOnlyDictionary MemberNamesAndDocs { get; } + + public override bool Equals(object? obj) => this.Equals(obj as DocEnum); + + public override int GetHashCode() + { + unchecked + { + int hash = this.IsFlags ? 1 : 0; + foreach (KeyValuePair entry in this.MemberNamesAndDocs) + { + hash += entry.Key.GetHashCode(); + hash += entry.Value?.GetHashCode() ?? 0; + } + + return hash; + } + } + + public bool Equals(DocEnum? other) + { + if (other is null) + { + return false; + } + + if (this.IsFlags != other.IsFlags) + { + return false; + } + + if (this.MemberNamesAndDocs.Count != other.MemberNamesAndDocs.Count) + { + return false; + } + + foreach (KeyValuePair entry in this.MemberNamesAndDocs) + { + if (!other.MemberNamesAndDocs.TryGetValue(entry.Key, out string? value)) + { + return false; + } + + if (entry.Value != value) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/ScrapeDocs/Program.cs b/src/ScrapeDocs/Program.cs index 02a70f13..33463852 100644 --- a/src/ScrapeDocs/Program.cs +++ b/src/ScrapeDocs/Program.cs @@ -5,6 +5,8 @@ namespace ScrapeDocs { using System; using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; @@ -27,6 +29,7 @@ internal class Program private static readonly Regex ReturnHeaderPattern = new Regex(@"^## -returns", RegexOptions.Compiled); private static readonly Regex RemarksHeaderPattern = new Regex(@"^## -remarks", RegexOptions.Compiled); private static readonly Regex InlineCodeTag = new Regex(@"\(.*)\", RegexOptions.Compiled); + private static readonly Regex EnumNameCell = new Regex(@"\]*\>\ results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> documentedEnums) + { + var uniqueEnums = new Dictionary>(); + var constantsDocs = new Dictionary>(); + foreach (var item in documentedEnums) + { + if (!uniqueEnums.TryGetValue(item.Value, out List<(string MethodName, string ParameterName, string HelpLink)>? list)) + { + uniqueEnums.Add(item.Value, list = new()); + } + + list.Add(item.Key); + + foreach (KeyValuePair enumValue in item.Value.MemberNamesAndDocs) + { + if (enumValue.Value is object) + { + if (!constantsDocs.TryGetValue(enumValue.Key, out List<(string MethodName, string HelpLink, string Doc)>? values)) + { + constantsDocs.Add(enumValue.Key, values = new()); + } + + values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value)); + } + } + } + + foreach (var item in constantsDocs) + { + string doc = item.Value[0].Doc; + + // If the documentation varies across methods, just link to each document. + bool differenceDetected = false; + for (int i = 1; i < item.Value.Count; i++) + { + if (item.Value[i].Doc != doc) + { + differenceDetected = true; + break; + } + } + + var docNode = new YamlMappingNode(); + if (differenceDetected) + { + doc = "Documentation varies per use. Refer to each: " + string.Join(", ", item.Value.Select(v => @$"{v.MethodName}")) + "."; + } + else + { + // Just point to any arbitrary method that documents it. + docNode.Add("HelpLink", item.Value[0].HelpLink); + } + + docNode.Add("Description", doc); + + // Skip the NULL constant due to https://github.com/aaubry/YamlDotNet/issues/591. + // Besides--documenting null probably isn't necessary. + if (item.Key != "NULL") + { + results.TryAdd(new YamlScalarNode(item.Key), docNode); + } + } + + return constantsDocs.Count; + } + private void Worker(CancellationToken cancellationToken) { Console.WriteLine("Enumerating documents to be parsed..."); string[] paths = Directory.GetFiles(this.contentBasePath, "??-*-*.md", SearchOption.AllDirectories) - ////.Where(p => Path.GetFileNameWithoutExtension(p).Contains("lastinputinfo")).ToArray() + ////.Where(p => p.Contains(@"ext\sdk-api\sdk-api-src\content\winuser\nf-winuser-setwindowpos.md")).ToArray() ; Console.WriteLine("Parsing documents..."); var timer = Stopwatch.StartNew(); var parsedNodes = from path in paths.AsParallel() let result = this.ParseDocFile(path) - where result is { } - select (Path: path, result.Value.ApiName, result.Value.YamlNode); + where result is not null + select (Path: path, result.Value.ApiName, result.Value.YamlNode, result.Value.EnumsByParameter); var results = new ConcurrentDictionary(); + var documentedEnums = new ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum>(); if (Debugger.IsAttached) { parsedNodes = parsedNodes.WithDegreeOfParallelism(1); // improve debuggability } parsedNodes - .WithCancellation(cancellationToken) - .ForAll(result => results.TryAdd(new YamlScalarNode(result.ApiName), result.YamlNode)); - Console.WriteLine("Parsed {2} documents in {0} ({1} per document)", timer.Elapsed, timer.Elapsed / paths.Length, paths.Length); + .WithCancellation<(string Path, string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter)>(cancellationToken) + .ForAll(result => + { + results.TryAdd(new YamlScalarNode(result.ApiName), result.YamlNode); + foreach (var e in result.EnumsByParameter) + { + string helpLink = ((YamlScalarNode)result.YamlNode["HelpLink"]).Value!; + documentedEnums.TryAdd((result.ApiName, e.Key, helpLink), e.Value); + } + }); + if (paths.Length == 0) + { + Console.Error.WriteLine("No documents found to parse."); + } + else + { + Console.WriteLine("Parsed {2} documents in {0} ({1} per document)", timer.Elapsed, timer.Elapsed / paths.Length, paths.Length); + Console.WriteLine($"Found {documentedEnums.Count} enums."); + } + + Console.WriteLine("Analyzing and naming enums and collecting docs on their members..."); + int constantsCount = AnalyzeEnums(results, documentedEnums); + Console.WriteLine($"Found docs for {constantsCount} constants."); Console.WriteLine("Writing results to \"{0}\"", this.outputPath); var yamlDocument = new YamlDocument(new YamlMappingNode(results)); @@ -108,199 +198,312 @@ where result is { } yamlStream.Save(yamlWriter); } - private (string ApiName, YamlNode YamlNode)? ParseDocFile(string filePath) + private (string ApiName, YamlNode YamlNode, IReadOnlyDictionary EnumsByParameter)? ParseDocFile(string filePath) { - var yaml = new YamlStream(); - using StreamReader mdFileReader = File.OpenText(filePath); - using var markdownToYamlReader = new YamlSectionReader(mdFileReader); - var yamlBuilder = new StringBuilder(); - string? line; - while ((line = markdownToYamlReader.ReadLine()) is object) - { - yamlBuilder.AppendLine(line); - } - try { - yaml.Load(new StringReader(yamlBuilder.ToString())); - } - catch (YamlDotNet.Core.YamlException ex) - { - Debug.WriteLine("YAML parsing error in \"{0}\": {1}", filePath, ex.Message); - return null; - } + IDictionary? enumsByParameter = null; + var yaml = new YamlStream(); + using StreamReader mdFileReader = File.OpenText(filePath); + using var markdownToYamlReader = new YamlSectionReader(mdFileReader); + var yamlBuilder = new StringBuilder(); + string? line; + while ((line = markdownToYamlReader.ReadLine()) is object) + { + yamlBuilder.AppendLine(line); + } - YamlSequenceNode methodNames = (YamlSequenceNode)yaml.Documents[0].RootNode["api_name"]; - bool TryGetProperName(string searchFor, char? suffix, [NotNullWhen(true)] out string? match) - { - if (suffix.HasValue) + try { - if (searchFor.EndsWith(suffix.Value)) + yaml.Load(new StringReader(yamlBuilder.ToString())); + } + catch (YamlDotNet.Core.YamlException ex) + { + Debug.WriteLine("YAML parsing error in \"{0}\": {1}", filePath, ex.Message); + return null; + } + + YamlSequenceNode methodNames = (YamlSequenceNode)yaml.Documents[0].RootNode["api_name"]; + bool TryGetProperName(string searchFor, char? suffix, [NotNullWhen(true)] out string? match) + { + if (suffix.HasValue) { - searchFor = searchFor.Substring(0, searchFor.Length - 1); + if (searchFor.EndsWith(suffix.Value)) + { + searchFor = searchFor.Substring(0, searchFor.Length - 1); + } + else + { + match = null; + return false; + } } - else + + match = methodNames.Children.Cast().FirstOrDefault(c => string.Equals(c.Value?.Replace('.', '-'), searchFor, StringComparison.OrdinalIgnoreCase))?.Value; + + if (suffix.HasValue && match is object) { - match = null; - return false; + match += char.ToUpper(suffix.Value, CultureInfo.InvariantCulture); } + + return match is object; } - match = methodNames.Children.Cast().FirstOrDefault(c => string.Equals(c.Value?.Replace('.', '-'), searchFor, StringComparison.OrdinalIgnoreCase))?.Value; + string presumedMethodName = FileNamePattern.Match(Path.GetFileNameWithoutExtension(filePath)).Groups[1].Value; - if (suffix.HasValue && match is object) + // Some structures have filenames that include the W or A suffix when the content doesn't. So try some fuzzy matching. + if (!TryGetProperName(presumedMethodName, null, out string? properName) && + !TryGetProperName(presumedMethodName, 'a', out properName) && + !TryGetProperName(presumedMethodName, 'w', out properName)) { - match += char.ToUpper(suffix.Value, CultureInfo.InvariantCulture); + Debug.WriteLine("WARNING: Could not find proper API name in: {0}", filePath); + return null; } - return match is object; - } + var methodNode = new YamlMappingNode(); + Uri helpLink = new Uri("https://docs.microsoft.com/windows/win32/api/" + filePath.Substring(this.contentBasePath.Length, filePath.Length - 3 - this.contentBasePath.Length).Replace('\\', '/')); + methodNode.Add("HelpLink", helpLink.AbsoluteUri); - string presumedMethodName = FileNamePattern.Match(Path.GetFileNameWithoutExtension(filePath)).Groups[1].Value; + var description = ((YamlMappingNode)yaml.Documents[0].RootNode).Children.FirstOrDefault(n => n.Key is YamlScalarNode { Value: "description" }).Value as YamlScalarNode; + if (description is object) + { + methodNode.Add("Description", description); + } - // Some structures have filenames that include the W or A suffix when the content doesn't. So try some fuzzy matching. - if (!TryGetProperName(presumedMethodName, null, out string? properName) && - !TryGetProperName(presumedMethodName, 'a', out properName) && - !TryGetProperName(presumedMethodName, 'w', out properName)) - { - Debug.WriteLine("WARNING: Could not find proper API name in: {0}", filePath); - return null; - } + // Search for parameter/field docs + var parametersMap = new YamlMappingNode(); + var fieldsMap = new YamlMappingNode(); + YamlScalarNode? remarksNode = null; + StringBuilder docBuilder = new StringBuilder(); + line = mdFileReader.ReadLine(); - var methodNode = new YamlMappingNode(); - Uri helpLink = new Uri("https://docs.microsoft.com/windows/win32/api/" + filePath.Substring(this.contentBasePath.Length, filePath.Length - 3 - this.contentBasePath.Length).Replace('\\', '/')); - methodNode.Add("HelpLink", helpLink.AbsoluteUri); + static string FixupLine(string line) + { + line = line.Replace("href=\"/", "href=\"https://docs.microsoft.com/"); + line = InlineCodeTag.Replace(line, match => $"{match.Groups[1].Value}"); + return line; + } - var description = ((YamlMappingNode)yaml.Documents[0].RootNode).Children.FirstOrDefault(n => n.Key is YamlScalarNode { Value: "description" }).Value as YamlScalarNode; - if (description is object) - { - methodNode.Add("Description", description); - } + void ParseTextSection(out YamlScalarNode node) + { + while ((line = mdFileReader.ReadLine()) is object) + { + if (line.StartsWith('#')) + { + break; + } - // Search for parameter/field docs - var parametersMap = new YamlMappingNode(); - var fieldsMap = new YamlMappingNode(); - YamlScalarNode? remarksNode = null; - StringBuilder docBuilder = new StringBuilder(); - line = mdFileReader.ReadLine(); + line = FixupLine(line); + docBuilder.AppendLine(line); + } - static string FixupLine(string line) - { - line = line.Replace("href=\"/", "href=\"https://docs.microsoft.com/"); - line = InlineCodeTag.Replace(line, match => $"{match.Groups[1].Value}"); - return line; - } + node = new YamlScalarNode(docBuilder.ToString()); - void ParseTextSection(out YamlScalarNode node) - { - while ((line = mdFileReader.ReadLine()) is object) + docBuilder.Clear(); + } + + IReadOnlyDictionary ParseEnumTable() { - if (line.StartsWith('#')) + var enums = new Dictionary(); + int state = 0; + const int StateReadingHeader = 0; + const int StateReadingName = 1; + const int StateLookingForDocColumn = 2; + const int StateReadingDocColumn = 3; + string? enumName = null; + var docsBuilder = new StringBuilder(); + while ((line = mdFileReader.ReadLine()) is object) { - break; + if (line == "") + { + break; + } + + switch (state) + { + case StateReadingHeader: + // Reading TR header + if (line == "") + { + state = StateReadingName; + } + + break; + + case StateReadingName: + // Reading an enum row's name column. + Match m = EnumNameCell.Match(line); + if (m.Success) + { + enumName = m.Groups[1].Value; + state = StateLookingForDocColumn; + } + + break; + + case 2: + // Looking for an enum row's doc column. + if (line.StartsWith("", StringComparison.OrdinalIgnoreCase)) + { + // The row ended before we found the doc column. + state = StateReadingName; + enums.Add(enumName!, null); + enumName = null; + } + + break; + + case 3: + // Reading the enum row's doc column. + if (line.StartsWith("", StringComparison.OrdinalIgnoreCase)) + { + state = StateReadingName; + + // Some docs are invalid in documenting the same enum multiple times. + if (!enums.ContainsKey(enumName!)) + { + enums.Add(enumName!, docsBuilder.ToString().Trim()); + } + + enumName = null; + docsBuilder.Clear(); + break; + } + + docsBuilder.AppendLine(FixupLine(line)); + break; + } } - line = FixupLine(line); - docBuilder.AppendLine(line); + return enums; } - node = new YamlScalarNode(docBuilder.ToString()); + void ParseSection(Match match, YamlMappingNode receivingMap, bool lookForEnums = false) + { + string sectionName = match.Groups[1].Value; + bool foundEnum = false; + bool foundEnumIsFlags = false; + while ((line = mdFileReader.ReadLine()) is object) + { + if (line.StartsWith('#')) + { + break; + } - docBuilder.Clear(); - } + if (lookForEnums) + { + if (foundEnum) + { + if (line == "") + { + IReadOnlyDictionary enumNamesAndDocs = ParseEnumTable(); + enumsByParameter ??= new Dictionary(); + enumsByParameter.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); + lookForEnums = false; + } + } + else + { + foundEnum = line.Contains("of the following values", StringComparison.OrdinalIgnoreCase); + foundEnumIsFlags = line.Contains("combination of", StringComparison.OrdinalIgnoreCase); + } + } - void ParseSection(Match match, YamlMappingNode receivingMap) - { - string sectionName = match.Groups[1].Value; - while ((line = mdFileReader.ReadLine()) is object) - { - if (line.StartsWith('#')) + if (!foundEnum) + { + line = FixupLine(line); + docBuilder.AppendLine(line); + } + } + + try + { + receivingMap.Add(sectionName, docBuilder.ToString().Trim()); + } + catch (ArgumentException) { - break; } - line = FixupLine(line); - docBuilder.AppendLine(line); + docBuilder.Clear(); } - try - { - receivingMap.Add(sectionName, docBuilder.ToString().Trim()); - } - catch (ArgumentException) + while (line is object) { - } - - docBuilder.Clear(); - } + if (ParameterHeaderPattern.Match(line) is Match { Success: true } parameterMatch) + { + ParseSection(parameterMatch, parametersMap, lookForEnums: true); + } + else if (FieldHeaderPattern.Match(line) is Match { Success: true } fieldMatch) + { + ParseSection(fieldMatch, fieldsMap); + } + else if (RemarksHeaderPattern.Match(line) is Match { Success: true } remarksMatch) + { + ParseTextSection(out remarksNode); + } + else + { + if (line is object && ReturnHeaderPattern.IsMatch(line)) + { + break; + } - while (line is object) - { - if (ParameterHeaderPattern.Match(line) is Match { Success: true } parameterMatch) - { - ParseSection(parameterMatch, parametersMap); + line = mdFileReader.ReadLine(); + } } - else if (FieldHeaderPattern.Match(line) is Match { Success: true } fieldMatch) + + if (parametersMap.Any()) { - ParseSection(fieldMatch, fieldsMap); + methodNode.Add("Parameters", parametersMap); } - else if (RemarksHeaderPattern.Match(line) is Match { Success: true } remarksMatch) + + if (fieldsMap.Any()) { - ParseTextSection(out remarksNode); + methodNode.Add("Fields", fieldsMap); } - else - { - if (line is object && ReturnHeaderPattern.IsMatch(line)) - { - break; - } - line = mdFileReader.ReadLine(); + if (remarksNode is object) + { + methodNode.Add("Remarks", remarksNode); } - } - - if (parametersMap.Any()) - { - methodNode.Add("Parameters", parametersMap); - } - - if (fieldsMap.Any()) - { - methodNode.Add("Fields", fieldsMap); - } - if (remarksNode is object) - { - methodNode.Add("Remarks", remarksNode); - } - - // Search for return value documentation - while (line is object) - { - Match m = ReturnHeaderPattern.Match(line); - if (m.Success) + // Search for return value documentation + while (line is object) { - while ((line = mdFileReader.ReadLine()) is object) + Match m = ReturnHeaderPattern.Match(line); + if (m.Success) { - if (line.StartsWith('#')) + while ((line = mdFileReader.ReadLine()) is object) { - break; + if (line.StartsWith('#')) + { + break; + } + + docBuilder.AppendLine(line); } - docBuilder.AppendLine(line); + methodNode.Add("ReturnValue", docBuilder.ToString().Trim()); + docBuilder.Clear(); + break; + } + else + { + line = mdFileReader.ReadLine(); } - - methodNode.Add("ReturnValue", docBuilder.ToString().Trim()); - docBuilder.Clear(); - break; - } - else - { - line = mdFileReader.ReadLine(); } - } - return (properName, methodNode); + enumsByParameter ??= ImmutableDictionary.Empty; + return (properName, methodNode, (IReadOnlyDictionary)enumsByParameter); + } + catch (Exception ex) + { + throw new ApplicationException($"Failed parsing \"{filePath}\".", ex); + } } private class YamlSectionReader : TextReader