Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor readers to reduce surface area #1975

Merged
merged 55 commits into from
Dec 20, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
a648591
Refactor readers to reduce surface area
darrelmiller Nov 26, 2024
c9b01cb
Moved load external references
darrelmiller Nov 26, 2024
634cb1c
Merge remote-tracking branch 'origin/vnext' into mk/fix-json-reader
MaggieKimani1 Nov 27, 2024
1464e2c
Move load methods to be adjacent
MaggieKimani1 Nov 27, 2024
e588377
Async over sync
MaggieKimani1 Nov 27, 2024
47e3e25
refactor code
MaggieKimani1 Nov 27, 2024
8521082
Use the provided format in hidi options
MaggieKimani1 Nov 28, 2024
99a80b9
Clean up tests; refactor to use async
MaggieKimani1 Nov 28, 2024
627d75b
Dispose stream
MaggieKimani1 Nov 28, 2024
46718f0
clean up and dispose stream if specified in the settings
MaggieKimani1 Nov 28, 2024
bd64612
Update public API
MaggieKimani1 Nov 28, 2024
a2f70ae
Leave stream open if specified in settings
MaggieKimani1 Nov 28, 2024
8bbb3dd
Guard against null references
MaggieKimani1 Nov 28, 2024
cecf011
code cleanup
MaggieKimani1 Nov 28, 2024
be3bbf5
Update public API
MaggieKimani1 Dec 2, 2024
408e793
remove extra semi-colon and commented out code
MaggieKimani1 Dec 2, 2024
7acdf9b
Remove unnecessary using
MaggieKimani1 Dec 2, 2024
dc25f9e
Inspect string input's format if not explicitly provided
MaggieKimani1 Dec 3, 2024
63ab53f
Remove unnecessary using and whitespace
MaggieKimani1 Dec 3, 2024
d022fa4
Remove unnecessary using
MaggieKimani1 Dec 5, 2024
04af1a6
Make format optional; add logic for inspecting stream format
MaggieKimani1 Dec 5, 2024
86b70c9
code cleanup
MaggieKimani1 Dec 5, 2024
dd80ab7
Update public API
MaggieKimani1 Dec 10, 2024
2c2bbe6
chore: code fixes recommended by sonarqube
baywet Dec 11, 2024
3665930
Update src/Microsoft.OpenApi.Readers/OpenApiYamlReader.cs
MaggieKimani1 Dec 11, 2024
5ca061b
Update src/Microsoft.OpenApi/Models/OpenApiDocument.cs
MaggieKimani1 Dec 11, 2024
e8c76db
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 11, 2024
a66e21b
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 11, 2024
4198c82
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 11, 2024
0836e97
chore: code linting
baywet Dec 11, 2024
a5e1057
Update src/Microsoft.OpenApi/Reader/OpenApiJsonReader.cs
MaggieKimani1 Dec 11, 2024
01e4f49
chore: adds missing defensive programming and passes settings when re…
baywet Dec 11, 2024
d1aedb4
Merge branch 'mk/fix-json-reader' of https://github.com/microsoft/Ope…
baywet Dec 11, 2024
07ab67a
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 11, 2024
3123cb7
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 11, 2024
ab2ddf0
fix: default settings in case of null value
baywet Dec 11, 2024
87ce841
Merge branch 'mk/fix-json-reader' of https://github.com/microsoft/Ope…
baywet Dec 11, 2024
3154d45
Merge branch 'vnext' into mk/fix-json-reader
MaggieKimani1 Dec 19, 2024
955f7fb
Fix issues from resolving merge conflicts
MaggieKimani1 Dec 19, 2024
a4933ef
Update input doc comments
MaggieKimani1 Dec 19, 2024
538d2eb
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 19, 2024
f879452
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 19, 2024
aac99fe
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 19, 2024
e3049a6
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 19, 2024
8a17bf2
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Dec 19, 2024
c944a31
Defensive programming; pass cancellation token
MaggieKimani1 Dec 20, 2024
66cc6e0
Merge remote-tracking branch 'origin/mk/fix-json-reader' into mk/fix-…
MaggieKimani1 Dec 20, 2024
47dca99
Code cleanup and fix tests
MaggieKimani1 Dec 20, 2024
e243f4e
simplify tuple variables
MaggieKimani1 Dec 20, 2024
28e1ecd
Update public API
MaggieKimani1 Dec 20, 2024
56742b9
Dispose stream in caller method
MaggieKimani1 Dec 20, 2024
ee63758
Abstract implementation detail from interface
MaggieKimani1 Dec 20, 2024
7270a89
Dispose preparedStream properly
MaggieKimani1 Dec 20, 2024
b033b76
Pass cancellation token
MaggieKimani1 Dec 20, 2024
c24d670
Update API
MaggieKimani1 Dec 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Microsoft.OpenApi.Readers/OpenApiYamlReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public ReadResult Read(MemoryStream input,
if (settings is null) throw new ArgumentNullException(nameof(settings));
JsonNode jsonNode;

// Parse the YAML text in the TextReader into a sequence of JsonNodes
// Parse the YAML text in the stream into a sequence of JsonNodes
try
{
using var stream = new StreamReader(input, default, true, -1, settings.LeaveStreamOpen);
Fixed Show fixed Hide fixed
Expand Down
6 changes: 3 additions & 3 deletions src/Microsoft.OpenApi/Interfaces/IOpenApiReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public interface IOpenApiReader
/// <summary>
/// Async method to reads the stream and parse it into an Open API document.
/// </summary>
/// <param name="input">The TextReader input.</param>
/// <param name="input">The stream input.</param>
/// <param name="settings"> The OpenApi reader settings.</param>
/// <param name="cancellationToken">Propagates notification that an operation should be cancelled.</param>
/// <returns></returns>
Expand All @@ -43,7 +43,7 @@ public interface IOpenApiReader
/// <summary>
/// Reads the MemoryStream and parses the fragment of an OpenAPI description into an Open API Element.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="input">Memory stream containing OpenAPI description to parse.</param>
/// <param name="version">Version of the OpenAPI specification that the fragment conforms to.</param>
/// <param name="diagnostic">Returns diagnostic object containing errors detected during parsing.</param>
/// <param name="settings">The OpenApiReader settings.</param>
Expand All @@ -53,7 +53,7 @@ public interface IOpenApiReader
/// <summary>
baywet marked this conversation as resolved.
Show resolved Hide resolved
/// Reads the JsonNode input and parses the fragment of an OpenAPI description into an Open API Element.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="input">Memory stream containing OpenAPI description to parse.</param>
/// <param name="version">Version of the OpenAPI specification that the fragment conforms to.</param>
/// <param name="diagnostic">Returns diagnostic object containing errors detected during parsing.</param>
/// <param name="settings">The OpenApiReader settings.</param>
Expand Down
23 changes: 16 additions & 7 deletions src/Microsoft.OpenApi/Reader/OpenApiJsonReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using Microsoft.OpenApi.Validations;
using System.Linq;
using Microsoft.OpenApi.Interfaces;
using System;

namespace Microsoft.OpenApi.Reader
{
Expand All @@ -23,17 +24,20 @@ public class OpenApiJsonReader : IOpenApiReader
/// <summary>
/// Reads the memory stream input and parses it into an Open API document.
/// </summary>
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="input">Memory stream containing OpenAPI description to parse.</param>
/// <param name="settings">The Reader settings to be used during parsing.</param>
/// <returns></returns>
public ReadResult Read(MemoryStream input,
OpenApiReaderSettings settings)
{
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
if (input is null) throw new ArgumentNullException(nameof(input));
if (settings is null) throw new ArgumentNullException(nameof(settings));

JsonNode jsonNode;
var diagnostic = new OpenApiDiagnostic();
settings ??= new OpenApiReaderSettings();

// Parse the JSON text in the TextReader into JsonNodes
// Parse the JSON text in the stream into JsonNodes
try
{
jsonNode = JsonNode.Parse(input);
Expand Down Expand Up @@ -62,6 +66,9 @@ public ReadResult Read(JsonNode jsonNode,
OpenApiReaderSettings settings,
string format = null)
{
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
if (jsonNode is null) throw new ArgumentNullException(nameof(jsonNode));
if (settings is null) throw new ArgumentNullException(nameof(settings));

var diagnostic = new OpenApiDiagnostic();
var context = new ParsingContext(diagnostic)
{
Expand Down Expand Up @@ -106,19 +113,21 @@ public ReadResult Read(JsonNode jsonNode,
/// <summary>
/// Reads the stream input asynchronously and parses it into an Open API document.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="input">Memory stream containing OpenAPI description to parse.</param>
/// <param name="settings">The Reader settings to be used during parsing.</param>
/// <param name="cancellationToken">Propagates notifications that operations should be cancelled.</param>
/// <returns></returns>
public async Task<ReadResult> ReadAsync(Stream input,
OpenApiReaderSettings settings,
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
CancellationToken cancellationToken = default)
{
if (input is null) throw new ArgumentNullException(nameof(input));
if (settings is null) throw new ArgumentNullException(nameof(settings));

JsonNode jsonNode;
var diagnostic = new OpenApiDiagnostic();
settings ??= new OpenApiReaderSettings();

// Parse the JSON text in the TextReader into JsonNodes
// Parse the JSON text in the stream into JsonNodes
try
{
jsonNode = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken).ConfigureAwait(false);
baywet marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -142,6 +151,8 @@ public T ReadFragment<T>(MemoryStream input,
out OpenApiDiagnostic diagnostic,
OpenApiReaderSettings settings = null) where T : IOpenApiElement
{
if (input is null) throw new ArgumentNullException(nameof(input));

JsonNode jsonNode;

// Parse the JSON
Expand Down Expand Up @@ -195,7 +206,5 @@ public T ReadFragment<T>(JsonNode input,

return (T)element;
}


}
}
92 changes: 51 additions & 41 deletions src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
string format = null,
OpenApiReaderSettings settings = null)
{
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
if (stream is null) throw new ArgumentNullException(nameof(stream));
settings ??= new OpenApiReaderSettings();

// Get the format of the stream if not provided
Expand All @@ -53,35 +54,6 @@
return result;
}

/// <summary>
/// Reads the stream input and ensures it is buffered before passing it to the Load method.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="input"></param>
/// <param name="version"></param>
/// <param name="format"></param>
/// <param name="diagnostic"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static T Load<T>(Stream input,
OpenApiSpecVersion version,
out OpenApiDiagnostic diagnostic,
string format = null,
OpenApiReaderSettings settings = null) where T : IOpenApiElement
{
if (input is MemoryStream memoryStream)
{
return Load<T>(memoryStream, version, format, out diagnostic, settings);
}
else
{
memoryStream = new MemoryStream();
input.CopyTo(memoryStream);
memoryStream.Position = 0;
return Load<T>(memoryStream, version, format, out diagnostic, settings);
}
}

/// <summary>
/// Reads the stream input and parses the fragment of an OpenAPI description into an Open API Element.
/// </summary>
Expand All @@ -104,11 +76,12 @@
/// </summary>
/// <param name="url">The path to the OpenAPI file</param>
/// <param name="settings"> The OpenApi reader settings.</param>
/// <param name="token"></param>
/// <returns></returns>
public static async Task<ReadResult> LoadAsync(string url, OpenApiReaderSettings settings = null)
public static async Task<ReadResult> LoadAsync(string url, OpenApiReaderSettings settings = null, CancellationToken token = default)
{
var result = await RetrieveStreamAndFormatAsync(url);
return await LoadAsync(result.Item1, result.Item2, settings).ConfigureAwait(false);
var result = await RetrieveStreamAndFormatAsync(url, token).ConfigureAwait(false);
return await LoadAsync(result.Item1, result.Item2, settings, token).ConfigureAwait(false);
}

/// <summary>
Expand All @@ -118,12 +91,13 @@
/// <param name="url">The path to the OpenAPI file</param>
/// <param name="version">Version of the OpenAPI specification that the fragment conforms to.</param>
/// <param name="settings">The OpenApiReader settings.</param>
/// <param name="token"></param>
/// <returns>Instance of newly created IOpenApiElement.</returns>
/// <returns>The OpenAPI element.</returns>
public static async Task<T> LoadAsync<T>(string url, OpenApiSpecVersion version, OpenApiReaderSettings settings = null) where T : IOpenApiElement
public static async Task<T> LoadAsync<T>(string url, OpenApiSpecVersion version, OpenApiReaderSettings settings = null, CancellationToken token = default) where T : IOpenApiElement
{
var result = await RetrieveStreamAndFormatAsync(url).ConfigureAwait(false);
return Load<T>(result.Item1, version, out var _, result.Item2, settings);
var result = await RetrieveStreamAndFormatAsync(url, token).ConfigureAwait(false);
return await LoadAsync<T>(result.Item1, version, result.Item2, settings);
baywet marked this conversation as resolved.
Show resolved Hide resolved
}

/// <summary>
Expand All @@ -136,7 +110,9 @@
/// <returns></returns>
public static async Task<ReadResult> LoadAsync(Stream input, string format = null, OpenApiReaderSettings settings = null, CancellationToken cancellationToken = default)
Fixed Show fixed Hide fixed
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
{
if (input is null) throw new ArgumentNullException(nameof(input));
settings ??= new OpenApiReaderSettings();

Stream preparedStream;
if (format is null)
{
Expand All @@ -150,7 +126,7 @@
}

// Use StreamReader to process the prepared stream (buffered for YAML, direct for JSON)
var result = await InternalLoadAsync(preparedStream, format, settings, cancellationToken);
var result = await InternalLoadAsync(preparedStream, format, settings, cancellationToken).ConfigureAwait(false);
if (!settings.LeaveStreamOpen)
{
input.Dispose();
Expand All @@ -159,6 +135,34 @@
return result;
}

/// <summary>
/// Reads the stream input and ensures it is buffered before passing it to the Load method.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="input"></param>
/// <param name="version"></param>
/// <param name="format"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static async Task<T> LoadAsync<T>(Stream input,
OpenApiSpecVersion version,
string format = null,
OpenApiReaderSettings settings = null) where T : IOpenApiElement
baywet marked this conversation as resolved.
Show resolved Hide resolved
{
if (input is null) throw new ArgumentNullException(nameof(input));
if (input is MemoryStream memoryStream)
{
return Load<T>(memoryStream, version, format, out var _, settings);
}
else
{
memoryStream = new MemoryStream();
await input.CopyToAsync(memoryStream).ConfigureAwait(false);
memoryStream.Position = 0;
return Load<T>(memoryStream, version, format, out var _, settings);
}
}

/// <summary>
/// Reads the input string and parses it into an Open API document.
/// </summary>
Expand All @@ -170,6 +174,7 @@
string format = null,
OpenApiReaderSettings settings = null)
{
if (input is null) throw new ArgumentNullException(nameof(input));
format ??= InspectInputFormat(input);
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
settings ??= new OpenApiReaderSettings();

Expand All @@ -194,6 +199,7 @@
string format = null,
OpenApiReaderSettings settings = null) where T : IOpenApiElement
{
if (input is null) throw new ArgumentNullException(nameof(input));
format ??= InspectInputFormat(input);
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
settings ??= new OpenApiReaderSettings();
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(input));
Expand All @@ -209,7 +215,7 @@

if (settings?.LoadExternalRefs ?? DefaultReaderSettings.LoadExternalRefs)
{
var diagnosticExternalRefs = await LoadExternalRefsAsync(readResult.Document, cancellationToken, settings, format);
var diagnosticExternalRefs = await LoadExternalRefsAsync(readResult.Document, settings, format, cancellationToken).ConfigureAwait(false);
// Merge diagnostics of external reference
if (diagnosticExternalRefs != null)
{
Expand All @@ -221,7 +227,7 @@
return readResult;
}

private static async Task<OpenApiDiagnostic> LoadExternalRefsAsync(OpenApiDocument document, CancellationToken cancellationToken, OpenApiReaderSettings settings, string format = null)
private static async Task<OpenApiDiagnostic> LoadExternalRefsAsync(OpenApiDocument document, OpenApiReaderSettings settings, string format = null, CancellationToken token = default)
{
// Create workspace for all documents to live in.
var baseUrl = settings.BaseUrl ?? new Uri(OpenApiConstants.BaseRegistryUri);
Expand All @@ -230,7 +236,7 @@
// Load this root document into the workspace
var streamLoader = new DefaultStreamLoader(settings.BaseUrl);
var workspaceLoader = new OpenApiWorkspaceLoader(openApiWorkSpace, settings.CustomExternalLoader ?? streamLoader, settings);
return await workspaceLoader.LoadAsync(new OpenApiReference() { ExternalResource = "/" }, document, format ?? OpenApiConstants.Json, null, cancellationToken).ConfigureAwait(false);
return await workspaceLoader.LoadAsync(new OpenApiReference() { ExternalResource = "/" }, document, format ?? OpenApiConstants.Json, null, token).ConfigureAwait(false);
}

private static ReadResult InternalLoad(MemoryStream input, string format, OpenApiReaderSettings settings)
Expand All @@ -246,7 +252,7 @@
return readResult;
}

private static async Task<(Stream, string)> RetrieveStreamAndFormatAsync(string url)
private static async Task<(Stream, string)> RetrieveStreamAndFormatAsync(string url, CancellationToken token = default)
{
if (!string.IsNullOrEmpty(url))
{
Expand All @@ -256,11 +262,15 @@
if (url.StartsWith("http", StringComparison.OrdinalIgnoreCase)
|| url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
var response = await _httpClient.GetAsync(url);
var response = await _httpClient.GetAsync(url, token).ConfigureAwait(false);
var mediaType = response.Content.Headers.ContentType.MediaType;
var contentType = mediaType.Split(";".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)[0];
format = contentType.Split('/').LastOrDefault();
#if NETSTANDARD2_0
stream = await response.Content.ReadAsStreamAsync();
MaggieKimani1 marked this conversation as resolved.
Show resolved Hide resolved
#else
stream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false);;
#endif
return (stream, format);
}
else
Expand Down Expand Up @@ -341,7 +351,7 @@
else
{
// YAML or other non-JSON format; copy remaining input to a new stream.
preparedStream = new MemoryStream();
Dismissed Show dismissed Hide dismissed
bufferStream.Position = 0;
await bufferStream.CopyToAsync(preparedStream, 81920, token).ConfigureAwait(false); // Copy buffered portion
await input.CopyToAsync(preparedStream, 81920, token).ConfigureAwait(false); // Copy remaining data
Expand All @@ -355,7 +365,7 @@
if (!format.Equals(OpenApiConstants.Json, StringComparison.OrdinalIgnoreCase))
{
// Buffer stream for non-JSON formats (e.g., YAML) since they require synchronous reading
preparedStream = new MemoryStream();
Dismissed Show dismissed Hide dismissed
await input.CopyToAsync(preparedStream, 81920, token).ConfigureAwait(false);
preparedStream.Position = 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,12 @@ public async Task ParseBasicEncodingShouldSucceed()
}

[Fact]
public void ParseAdvancedEncodingShouldSucceed()
public async Task ParseAdvancedEncodingShouldSucceed()
{
using var stream = Resources.GetStream(Path.Combine(SampleFolderPath, "advancedEncoding.yaml"));

// Act
var encoding = OpenApiModelFactory.Load<OpenApiEncoding>(stream, OpenApiSpecVersion.OpenApi3_0, out _);
var encoding = await OpenApiModelFactory.LoadAsync<OpenApiEncoding>(stream, OpenApiSpecVersion.OpenApi3_0);

// Assert
encoding.Should().BeEquivalentTo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,12 @@ public async Task ParseBasicInfoShouldSucceed()
}

[Fact]
public void ParseMinimalInfoShouldSucceed()
public async Task ParseMinimalInfoShouldSucceed()
{
using var stream = Resources.GetStream(Path.Combine(SampleFolderPath, "minimalInfo.yaml"));

// Act
var openApiInfo = OpenApiModelFactory.Load<OpenApiInfo>(stream, OpenApiSpecVersion.OpenApi3_0, out _, "yaml");
var openApiInfo = await OpenApiModelFactory.LoadAsync<OpenApiInfo>(stream, OpenApiSpecVersion.OpenApi3_0);

// Assert
openApiInfo.Should().BeEquivalentTo(
Expand Down
Loading
Loading