OSOE-770: Add support for structured html-validate output

Lombiq.Tests.UI/Docs/Configuration.md

@@ -59,51 +59,22 @@ Recommendations and notes for such configuration:
 
 ### HTML validation configuration
 
-If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). For example:
-
-```json
-{
-  "extends": [
-    "html-validate:recommended"
-  ],
-
-  "rules": {
-    "attribute-boolean-style": "off",
-    "element-required-attributes": "off",
-    "no-trailing-whitespace": "off",
-    "no-inline-style": "off",
-    "no-implicit-button-type": "off",
-    "wcag/h30": "off",
-    "wcag/h32": "off",
-    "wcag/h36": "off",
-    "wcag/h37": "off",
-    "wcag/h67": "off",
-    "wcag/h71": "off"
-  },
-
-  "root":  true
-}
+If you want to change some HTML validation rules for only a few specific tests you can exclude them from the results.
+For example if you want to exclude the `prefer-native-element` rule from the results you can do it by doing the following:
+
+```c#
+configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationResultAsync =
+    validationResult =>
+    {
+        var errors = validationResult.GetParsedErrors()
+            .Where(error => error.RuleId is not "prefer-native-element");
+        errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message)));
+        return Task.CompletedTask;
+    });
 ```
 
-Then you can change the configuration to use that:
-
-```cs
- changeConfiguration: configuration => configuration.HtmlValidationConfiguration.HtmlValidationOptions =
-                configuration.HtmlValidationConfiguration.HtmlValidationOptions
-                    .CloneWith(validationOptions => validationOptions.ConfigPath =
-                        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json")));
-```
-
-Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this:
-
-```xml
-  <ItemGroup>
-    <Content Include="TestName.htmlvalidate.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <PackageCopyToOutput>true</PackageCopyToOutput>
-    </Content>
-  </ItemGroup>
- ```
+Note that the `RuleId` is the identifier of the rule that you want to exclude from the results.
+The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary.
 
 ## Multi-process test execution
 

Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs

@@ -1,8 +1,10 @@
 using Atata.HtmlValidation;
+using Lombiq.Tests.UI.Models;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Extensions;
@@ -22,4 +24,36 @@ public static async Task<IEnumerable<string>> GetErrorsAsync(this HtmlValidation
             .Select(error =>
                 (error.StartsWith("error:", StringComparison.OrdinalIgnoreCase) ? string.Empty : "error:") + error);
     }
+
+    /// <summary>
+    /// Gets the parsed errors from the HTML validation result.
+    /// Can only be used if the output formatter is set to JSON.
+    /// </summary>
+    public static IEnumerable<JsonHtmlValidationError> GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output);
+
+    private static IEnumerable<JsonHtmlValidationError> ParseOutput(string output)
+    {
+        output = output.Trim();
+        if ((!output.StartsWith('{') || !output.EndsWith('}')) &&
+            (!output.StartsWith('[') || !output.EndsWith(']')))
+        {
+            throw new JsonException($"Invalid JSON, make sure to set the OutputFormatter to JSON. Output: {output}");
+        }
+
+        try
+        {
+            var document = JsonDocument.Parse(output);
+            return document.RootElement.EnumerateArray()
+                .SelectMany(element => element.GetProperty("messages").EnumerateArray())
+                .Select(message =>
+                {
+                    var rawMessageText = message.GetRawText();
+                    return JsonSerializer.Deserialize<JsonHtmlValidationError>(rawMessageText);
+                });
+        }
+        catch (JsonException)
+        {
+            throw new JsonException("Unable to parse output, did you set the OutputFormatter to JSON?");
+        }
+    }
 }

Lombiq.Tests.UI/Extensions/HtmlValidationUITestContextExtensions.cs

@@ -1,4 +1,5 @@
 using Atata.Cli;
+using Atata.Cli.HtmlValidate;
 using Atata.HtmlValidation;
 using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Services;
@@ -53,6 +54,12 @@ public static HtmlValidationResult ValidateHtml(
     {
         var options = context.Configuration.HtmlValidationConfiguration.HtmlValidationOptions.Clone();
         htmlValidationOptionsAdjuster?.Invoke(options);
+
+        // Because of the default settings of Atata.HtmlValidation.HtmlValidationOptions overriding the formatter
+        // we need to set this back to JSON
+        if (options.OutputFormatter == HtmlValidateFormatter.Names.Stylish)
+            options.ResultFileFormatter = HtmlValidateFormatter.Names.Json;
+
         try
         {
             return new HtmlValidator(options).Validate(context.Driver.PageSource);

Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs

@@ -0,0 +1,28 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Lombiq.Tests.UI.Models;
+
+public class JsonHtmlValidationError
+{
+    [JsonPropertyName("ruleId")]
+    public string RuleId { get; set; }
+    [JsonPropertyName("severity")]
+    public int Severity { get; set; }
+    [JsonPropertyName("message")]
+    public string Message { get; set; }
+    [JsonPropertyName("offset")]
+    public int Offset { get; set; }
+    [JsonPropertyName("line")]
+    public int Line { get; set; }
+    [JsonPropertyName("column")]
+    public int Column { get; set; }
+    [JsonPropertyName("size")]
+    public int Size { get; set; }
+    [JsonPropertyName("selector")]
+    public string Selector { get; set; }
+    [JsonPropertyName("ruleUrl")]
+    public string RuleUrl { get; set; }
+    [JsonPropertyName("context")]
+    public JsonElement Context { get; set; }
+}

Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs

@@ -1,7 +1,10 @@
+using Atata.Cli.HtmlValidate;
 using Atata.HtmlValidation;
+using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Helpers;
 using Shouldly;
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Services;
@@ -28,6 +31,8 @@ public class HtmlValidationConfiguration
     /// </summary>
     public HtmlValidationOptions HtmlValidationOptions { get; set; } = new()
     {
+        ResultFileFormatter = HtmlValidateFormatter.Names.Text,
+        OutputFormatter = HtmlValidateFormatter.Names.Json,
         SaveHtmlToFile = HtmlSaveCondition.Never,
         SaveResultToFile = true,
         // This is necessary so no long folder names will be generated, see:
@@ -64,7 +69,18 @@ public class HtmlValidationConfiguration
     public static readonly Func<HtmlValidationResult, Task> AssertHtmlValidationOutputIsEmptyAsync =
         validationResult =>
         {
-            validationResult.Output.ShouldBeEmpty();
+            // Keep supporting cases where output format is not set to JSON.
+            if (validationResult.Output.Trim().StartsWith('[') ||
+                validationResult.Output.Trim().StartsWith('{'))
+            {
+                var errors = validationResult.GetParsedErrors();
+                errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message)));
+            }
+            else
+            {
+                validationResult.Output.ShouldBeEmpty();
+            }
+
             return Task.CompletedTask;
         };