From 267e324fdc33c904f6ec0df71c1041d5506de489 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 3 Jun 2020 16:28:28 +0200 Subject: [PATCH] Expose Serialize(StringBuilder) overload (#82) Co-authored-by: Rolf Kristensen --- src/Elastic.CommonSchema.NLog/EcsLayout.cs | 43 ++++----- .../Base.Serialization.cs | 55 +++--------- .../Elastic.CommonSchema.csproj | 2 +- .../Serialization/EcsSerializerFactory.cs | 45 ++++++++++ .../Serialization/ReusableUtf8JsonWriter.cs | 89 +++++++++++++++++++ .../Elastic.CommonSchema.Benchmarks.csproj | 10 +-- .../SerializingStringBuilderBase.cs | 70 +++++++++++++++ .../OutputTests.cs | 2 +- 8 files changed, 247 insertions(+), 69 deletions(-) create mode 100644 src/Elastic.CommonSchema/Serialization/EcsSerializerFactory.cs create mode 100644 src/Elastic.CommonSchema/Serialization/ReusableUtf8JsonWriter.cs create mode 100644 tests/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs diff --git a/src/Elastic.CommonSchema.NLog/EcsLayout.cs b/src/Elastic.CommonSchema.NLog/EcsLayout.cs index 7d49ab81..b2e19e77 100644 --- a/src/Elastic.CommonSchema.NLog/EcsLayout.cs +++ b/src/Elastic.CommonSchema.NLog/EcsLayout.cs @@ -117,42 +117,43 @@ public EcsLayout() [ArrayParameter(typeof(TargetPropertyWithContext), "tag")] public IList Tags { get; } = new List(); - protected override void RenderFormattedMessage(LogEventInfo logEventInfo, StringBuilder target) + protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuilder target) { var ecsEvent = new Base { - Timestamp = logEventInfo.TimeStamp, - Message = logEventInfo.FormattedMessage, + Timestamp = logEvent.TimeStamp, + Message = logEvent.FormattedMessage, Ecs = new Ecs { Version = Base.Version }, - Log = GetLog(logEventInfo), - Event = GetEvent(logEventInfo), - Metadata = GetMetadata(logEventInfo), - Process = GetProcess(logEventInfo), - Trace = GetTrace(logEventInfo), - Transaction = GetTransaction(logEventInfo), - Error = GetError(logEventInfo.Exception), - Tags = GetTags(logEventInfo), - Labels = GetLabels(logEventInfo), - Agent = GetAgent(logEventInfo), - Server = GetServer(logEventInfo), - Host = GetHost(logEventInfo) + Log = GetLog(logEvent), + Event = GetEvent(logEvent), + Metadata = GetMetadata(logEvent), + Process = GetProcess(logEvent), + Trace = GetTrace(logEvent), + Transaction = GetTransaction(logEvent), + Error = GetError(logEvent.Exception), + Tags = GetTags(logEvent), + Labels = GetLabels(logEvent), + Agent = GetAgent(logEvent), + Server = GetServer(logEvent), + Host = GetHost(logEvent) }; + //Give any deriving classes a chance to enrich the event - EnrichEvent(logEventInfo,ref ecsEvent); + EnrichEvent(logEvent, ref ecsEvent); //Allow programmatical actions to enrich before serializing - EnrichAction?.Invoke(ecsEvent, logEventInfo); - var output = ecsEvent.Serialize(); - target.Append(output); + EnrichAction?.Invoke(ecsEvent, logEvent); + + ecsEvent.Serialize(target); } /// /// Override to supplement the ECS event parsing /// - /// The original log event + /// The original log event /// The EcsEvent to modify /// Enriched ECS Event /// Destructive for performance - protected virtual void EnrichEvent(LogEventInfo logEventInfo,ref Base ecsEvent) + protected virtual void EnrichEvent(LogEventInfo logEvent, ref Base ecsEvent) { } diff --git a/src/Elastic.CommonSchema/Base.Serialization.cs b/src/Elastic.CommonSchema/Base.Serialization.cs index 7842acf7..0e6e3ddd 100644 --- a/src/Elastic.CommonSchema/Base.Serialization.cs +++ b/src/Elastic.CommonSchema/Base.Serialization.cs @@ -3,8 +3,8 @@ // See the LICENSE file in the project root for more information using System; -using System.Buffers; using System.IO; +using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Threading; @@ -12,45 +12,6 @@ using Elastic.CommonSchema.Serialization; using static Elastic.CommonSchema.Serialization.JsonConfiguration; -namespace Elastic.CommonSchema.Serialization -{ - /// - /// This static class allows you to deserialize subclasses of - /// If you are dealing with directly you do not need to use this class, - /// use and the overloads instead. - /// - /// - /// This class should only be used for advanced use cases, for simpler use cases you can utilise the property. - /// - /// Type of the subclass - public static class EcsSerializerFactory where TBase : Base, new() - { - public static ValueTask DeserializeAsync(Stream stream, CancellationToken ctx = default) => - JsonSerializer.DeserializeAsync(stream, SerializerOptions, ctx); - - public static TBase Deserialize(string json) => JsonSerializer.Deserialize(json, SerializerOptions); - - public static TBase Deserialize(ReadOnlySpan json) => JsonSerializer.Deserialize(json, SerializerOptions); - - public static TBase Deserialize(Stream stream) - { - using var ms = new MemoryStream(); - var buffer = ArrayPool.Shared.Rent(1024); - var total = 0; - int read; - while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) - { - ms.Write(buffer, 0, read); - total += read; - } - var span = ms.TryGetBuffer(out var segment) - ? new ReadOnlyMemory(segment.Array, segment.Offset, total).Span - : new ReadOnlyMemory(ms.ToArray()).Span; - return Deserialize(span); - } - } -} - namespace Elastic.CommonSchema { public partial class Base @@ -96,6 +57,16 @@ public static ValueTask DeserializeAsync(Stream stream, CancellationToken public byte[] SerializeToUtf8Bytes() => JsonSerializer.SerializeToUtf8Bytes(this, GetType(), SerializerOptions); + private static ReusableUtf8JsonWriter _reusableJsonWriter; + private static ReusableUtf8JsonWriter ReusableJsonWriter => _reusableJsonWriter ??= new ReusableUtf8JsonWriter(); + + public StringBuilder Serialize(StringBuilder stringBuilder) + { + using var reusableWriter = ReusableJsonWriter.AllocateJsonWriter(stringBuilder); + reusableWriter.Serialize(this); + return stringBuilder; + } + public void Serialize(Stream stream) { using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions @@ -103,9 +74,11 @@ public void Serialize(Stream stream) Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, Indented = false }); - JsonSerializer.Serialize(writer, this, JsonConfiguration.SerializerOptions); + JsonSerializer.Serialize(writer, this, SerializerOptions); } + internal void Serialize(Utf8JsonWriter writer) => JsonSerializer.Serialize(writer, this, SerializerOptions); + public Task SerializeAsync(Stream stream, CancellationToken ctx = default) => JsonSerializer.SerializeAsync(stream, this, GetType(), SerializerOptions, ctx); } diff --git a/src/Elastic.CommonSchema/Elastic.CommonSchema.csproj b/src/Elastic.CommonSchema/Elastic.CommonSchema.csproj index 3d14374d..f0aa3989 100644 --- a/src/Elastic.CommonSchema/Elastic.CommonSchema.csproj +++ b/src/Elastic.CommonSchema/Elastic.CommonSchema.csproj @@ -1,6 +1,6 @@  - + netstandard2.0;netstandard2.1;net461 Elastic Common Schema (ECS) Types diff --git a/src/Elastic.CommonSchema/Serialization/EcsSerializerFactory.cs b/src/Elastic.CommonSchema/Serialization/EcsSerializerFactory.cs new file mode 100644 index 00000000..9a84fb5b --- /dev/null +++ b/src/Elastic.CommonSchema/Serialization/EcsSerializerFactory.cs @@ -0,0 +1,45 @@ +using System; +using System.Buffers; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Elastic.CommonSchema.Serialization +{ + /// + /// This static class allows you to deserialize subclasses of + /// If you are dealing with directly you do not need to use this class, + /// use and the overloads instead. + /// + /// + /// This class should only be used for advanced use cases, for simpler use cases you can utilise the property. + /// + /// Type of the subclass + public static class EcsSerializerFactory where TBase : Base, new() + { + public static ValueTask DeserializeAsync(Stream stream, CancellationToken ctx = default) => + JsonSerializer.DeserializeAsync(stream, JsonConfiguration.SerializerOptions, ctx); + + public static TBase Deserialize(string json) => JsonSerializer.Deserialize(json, JsonConfiguration.SerializerOptions); + + public static TBase Deserialize(ReadOnlySpan json) => JsonSerializer.Deserialize(json, JsonConfiguration.SerializerOptions); + + public static TBase Deserialize(Stream stream) + { + using var ms = new MemoryStream(); + var buffer = ArrayPool.Shared.Rent(1024); + var total = 0; + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + { + ms.Write(buffer, 0, read); + total += read; + } + var span = ms.TryGetBuffer(out var segment) + ? new ReadOnlyMemory(segment.Array, segment.Offset, total).Span + : new ReadOnlyMemory(ms.ToArray()).Span; + return Deserialize(span); + } + } +} diff --git a/src/Elastic.CommonSchema/Serialization/ReusableUtf8JsonWriter.cs b/src/Elastic.CommonSchema/Serialization/ReusableUtf8JsonWriter.cs new file mode 100644 index 00000000..24e83d89 --- /dev/null +++ b/src/Elastic.CommonSchema/Serialization/ReusableUtf8JsonWriter.cs @@ -0,0 +1,89 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.IO; +using System.Text; +using System.Text.Json; + +namespace Elastic.CommonSchema.Serialization +{ + internal sealed class ReusableUtf8JsonWriter + { + private Utf8JsonWriter _cachedJsonWriter; + private readonly MemoryStream _cachedMemoryStream; + private readonly char[] _cachedEncodingBuffer; + + public ReusableUtf8JsonWriter() + { + _cachedMemoryStream = new MemoryStream(4 * 1024); + _cachedJsonWriter = new Utf8JsonWriter(_cachedMemoryStream); + _cachedEncodingBuffer = new char[1024]; + } + + public ReusableJsonWriter AllocateJsonWriter(StringBuilder text) + { + var writer = System.Threading.Interlocked.Exchange(ref _cachedJsonWriter, null); + return new ReusableJsonWriter(this, writer, text); + } + + public ReusableJsonWriter NewJsonWriter(StringBuilder text) => + new ReusableJsonWriter(this, new Utf8JsonWriter(new MemoryStream()), text); + + private void Return(Utf8JsonWriter writer, StringBuilder output) + { + writer.Flush(); + + if (_cachedMemoryStream.Length > 0) + { + if (!_cachedMemoryStream.TryGetBuffer(out var byteArray)) + byteArray = new ArraySegment(_cachedMemoryStream.GetBuffer(), 0, (int)_cachedMemoryStream.Length); + + CopyToStringBuilder(byteArray, _cachedEncodingBuffer, output); + } + + writer.Reset(); + _cachedMemoryStream.Position = 0; + _cachedMemoryStream.SetLength(0); + System.Threading.Interlocked.Exchange(ref _cachedJsonWriter, writer); + } + + private static void CopyToStringBuilder(ArraySegment byteArray, char[] encodingBuffer, StringBuilder output) + { + for (var i = 0; i < byteArray.Count; i += encodingBuffer.Length) + { + var byteCount = Math.Min(byteArray.Count - i, encodingBuffer.Length); + var charCount = Encoding.UTF8.GetChars(byteArray.Array, byteArray.Offset + i, byteCount, encodingBuffer, 0); + output.Append(encodingBuffer, 0, charCount); + } + } + + internal readonly struct ReusableJsonWriter : IDisposable + { + private readonly ReusableUtf8JsonWriter _owner; + private readonly Utf8JsonWriter _writer; + private readonly StringBuilder _output; + + public ReusableJsonWriter(ReusableUtf8JsonWriter owner, Utf8JsonWriter writer, StringBuilder output) + { + _writer = writer; + _owner = writer != null ? owner : null; + _output = output; + } + + public void Serialize(Base ecsEvent) + { + if (_writer != null) + ecsEvent.Serialize(_writer); + else + { + var result = ecsEvent.Serialize(); + _output.Append(result); + } + } + + public void Dispose() => _owner?.Return(_writer, _output); + } + } +} diff --git a/tests/Elastic.CommonSchema.Benchmarks/Elastic.CommonSchema.Benchmarks.csproj b/tests/Elastic.CommonSchema.Benchmarks/Elastic.CommonSchema.Benchmarks.csproj index 9910cfce..1300318c 100644 --- a/tests/Elastic.CommonSchema.Benchmarks/Elastic.CommonSchema.Benchmarks.csproj +++ b/tests/Elastic.CommonSchema.Benchmarks/Elastic.CommonSchema.Benchmarks.csproj @@ -1,5 +1,5 @@ - + Exe @@ -7,13 +7,13 @@ - - - + + + - + diff --git a/tests/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs b/tests/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs new file mode 100644 index 00000000..a9469d7a --- /dev/null +++ b/tests/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs @@ -0,0 +1,70 @@ +using System; +using System.Text; +using AutoBogus; +using BenchmarkDotNet.Attributes; + +namespace Elastic.CommonSchema.Benchmarks +{ + [UnicodeConsoleLogger, MemoryDiagnoser, ThreadingDiagnoser] + public class SerializingStringBuilderBase + { + [Benchmark] + public StringBuilder Empty() + { + var ecs = new Base(); + return ecs.Serialize(new StringBuilder()); + } + [Benchmark] + public StringBuilder Minimal() + { + var ecs = new Base + { + Timestamp = DateTimeOffset.UtcNow, + Log = new Log + { + Level = "Debug" + }, + Message = "hello world!" + }; + return ecs.Serialize(new StringBuilder()); + } + [Benchmark] + public StringBuilder Complex() + { + var ecs = new Base + { + Timestamp = DateTimeOffset.UtcNow, + Log = new Log + { + Level = "Debug", Logger = "Logger", + Origin = new LogOrigin + { + File = new OriginFile { Line = 12, Name = "file.cs"}, Function = "Complex" + }, + Original = "new log line", + Syslog = new LogSyslog { + Facility = new SyslogFacility + { + Code = 12, Name = "syslog" + }, Priority = 12, Severity = new SyslogSeverity() + { + Code = 12, Name = "asd" + }, + } + }, + Message = "hello world!", + Agent = new Agent + { + Name = "test" + } + + }; + return ecs.Serialize(new StringBuilder()); + } + + public static readonly Base FullInstance = new AutoFaker().Generate(); + + [Benchmark] + public StringBuilder Full() => FullInstance.Serialize(new StringBuilder()); + } +} diff --git a/tests/Elastic.CommonSchema.NLog.Tests/OutputTests.cs b/tests/Elastic.CommonSchema.NLog.Tests/OutputTests.cs index c2c314c6..08bdb67d 100644 --- a/tests/Elastic.CommonSchema.NLog.Tests/OutputTests.cs +++ b/tests/Elastic.CommonSchema.NLog.Tests/OutputTests.cs @@ -17,7 +17,7 @@ public void LogMultiple() => TestLogger((logger, getLogEvents) => { logger.Info("My log message!"); logger.Info("Test output to NLog!"); - Action sketchy = () => throw new Exception("I threw up."); + void sketchy() => throw new Exception("I threw up."); var exception = Record.Exception(sketchy); logger.Error(exception, "Here is an error."); Assert.NotNull(exception);