From 6bc15dc49ca520bec24d7f62cc87025a09336693 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 31 Dec 2024 09:53:35 +0000 Subject: [PATCH 01/11] feat(SystemTextJson): Compression --- CHANGELOG.md | 1 + src/FsCodec.SystemTextJson/Compression.fs | 139 ++++++++++++++++++ .../FsCodec.SystemTextJson.fsproj | 1 + .../CompressionTests.fs | 96 ++++++++++++ .../FsCodec.SystemTextJson.Tests/Examples.fsx | 23 +++ .../FsCodec.SystemTextJson.Tests.fsproj | 1 + 6 files changed, 261 insertions(+) create mode 100644 src/FsCodec.SystemTextJson/Compression.fs create mode 100644 tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs diff --git a/CHANGELOG.md b/CHANGELOG.md index f9df153..651e2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The `Unreleased` section name is replaced by the expected version of next releas ### Added - `Core.EventData.MapEx`: Enable contextual encoding of bodies [#127](https://github.com/jet/FsCodec/pull/127) +- `SystemTextJson.Compression`: Conditional compression as per `Box.Compression` [#126](https://github.com/jet/FsCodec/pull/126) ### Changed ### Removed diff --git a/src/FsCodec.SystemTextJson/Compression.fs b/src/FsCodec.SystemTextJson/Compression.fs new file mode 100644 index 0000000..d0a12fe --- /dev/null +++ b/src/FsCodec.SystemTextJson/Compression.fs @@ -0,0 +1,139 @@ +namespace FsCodec.SystemTextJson + +open FsCodec +open FsCodec.SystemTextJson.Interop +open System +open System.Runtime.CompilerServices +open System.Runtime.InteropServices +open System.Text.Json + +/// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value identifying the encoding scheme. +/// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used. +type EncodedBody = (struct(int * JsonElement)) + +module private EncodedMaybeCompressed = + + module Encoding = + let [] Direct = 0 // Assumed for all values not listed here + let [] Deflate = 1 // Deprecated encoding produced by Equinox.Cosmos/CosmosStore < v 4.1.0; no longer produced + let [] Brotli = 2 // Default encoding + + (* Decompression logic: triggered by extension methods below at the point where the Codec's Decode retrieves the Data or Meta properties *) + + // Equinox.Cosmos / Equinox.CosmosStore Deflate logic was as below: + // let private deflate (uncompressedBytes: byte[]) = + // let output = new MemoryStream() + // let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + // compressor.Write(uncompressedBytes) + // compressor.Flush() // Could `Close`, but not required + // output.ToArray() + let private inflate (compressedBytes: byte[]) = + let input = new System.IO.MemoryStream(compressedBytes) + let decompressor = new System.IO.Compression.DeflateStream(input, System.IO.Compression.CompressionMode.Decompress, leaveOpen = true) + let output = new System.IO.MemoryStream() + decompressor.CopyTo output + output.ToArray() + let private brotliDecompress (data: byte[]): byte[] = + let s = new System.IO.MemoryStream(data, writable = false) + use decompressor = new System.IO.Compression.BrotliStream(s, System.IO.Compression.CompressionMode.Decompress) + use output = new System.IO.MemoryStream() + decompressor.CopyTo output + output.ToArray() + let decodeJsonElement struct (encoding, data: JsonElement): JsonElement = + match encoding, data.ValueKind with + | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> inflate |> InteropHelpers.Utf8ToJsonElement + | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> brotliDecompress |> InteropHelpers.Utf8ToJsonElement + | _ -> data + let decodeUtf8 struct (encoding, data: JsonElement): ReadOnlyMemory = + match encoding, data.ValueKind with + | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> inflate |> ReadOnlyMemory + | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> brotliDecompress |> ReadOnlyMemory + | _ -> InteropHelpers.JsonElementToUtf8 data + + (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields + Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) + + let encodeUncompressed (raw: JsonElement): EncodedBody = Encoding.Direct, raw + let private blobToStringElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement + let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = + let output = new System.IO.MemoryStream() + use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + compressor.Write eventBody.Span + compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing + output + let tryCompress minSize minGain (raw: JsonElement): EncodedBody = + let utf8: ReadOnlyMemory = InteropHelpers.JsonElementToUtf8 raw + if utf8.Length < minSize then encodeUncompressed raw else + + let brotli = brotliCompress utf8 + if utf8.Length <= int brotli.Length + minGain then encodeUncompressed raw else + Encoding.Brotli, brotli.ToArray() |> blobToStringElement + let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw + let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = + if utf8.Length < minSize then encodeUncompressedUtf8 utf8 else + + let brotli = brotliCompress utf8 + if utf8.Length <= int brotli.Length + minGain then encodeUncompressedUtf8 utf8 else + Encoding.Brotli, brotli.ToArray() |> blobToStringElement + +type [] CompressionOptions = { minSize: int; minGain: int } with + /// Attempt to compress anything possible + // TL;DR in general it's worth compressing everything to minimize RU consumption both on insert and update + // For CosmosStore, every time we touch the tip, the RU impact of the write is significant, + // so preventing or delaying that is of critical importance + // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default + static member Default = { minSize = 48; minGain = 4 } + +[] +type Compression private () = + + static member Encode(x: JsonElement): EncodedBody = + EncodedMaybeCompressed.encodeUncompressed x + static member Encode(x: ReadOnlyMemory): EncodedBody = + EncodedMaybeCompressed.encodeUncompressedUtf8 x + static member EncodeTryCompress(options, x: JsonElement): EncodedBody = + EncodedMaybeCompressed.tryCompress options.minSize options.minGain x + static member EncodeTryCompress(options, x: ReadOnlyMemory): EncodedBody = + EncodedMaybeCompressed.tryCompressUtf8 options.minSize options.minGain x + static member DecodeToJsonElement(x: EncodedBody): JsonElement = + EncodedMaybeCompressed.decodeJsonElement x + static member DecodeToUtf8(x: EncodedBody): ReadOnlyMemory = + EncodedMaybeCompressed.decodeUtf8 x + static member DecodeToByteArray(x: EncodedBody): byte[] = + Compression.DecodeToUtf8(x).ToArray() + + /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
+ /// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
+ /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. + [] + static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) + : IEventCodec<'Event, EncodedBody, 'Context> = + let opts = defaultArg options CompressionOptions.Default + FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.EncodeTryCompress(opts, x)), Func<_, _> Compression.DecodeToUtf8) + + /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
+ /// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
+ /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. + [] + static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>, [] ?options) + : IEventCodec<'Event, EncodedBody, 'Context> = + let opts = defaultArg options CompressionOptions.Default + FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.EncodeTryCompress(opts, x)), Func<_, _> Compression.DecodeToJsonElement) + + /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. + [] + static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) + : IEventCodec<'Event, EncodedBody, 'Context> = + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Encode, Func<_, _> Compression.DecodeToJsonElement) + + /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. + [] + static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + : IEventCodec<'Event, ReadOnlyMemory, 'Context> = + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.DecodeToUtf8, Func<_, _> Compression.Encode) + + /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. + [] + static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + : IEventCodec<'Event, byte[], 'Context> = + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.DecodeToByteArray, Func<_, _> Compression.Encode) diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index ee35d82..7c55368 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -18,6 +18,7 @@ + diff --git a/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs b/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs new file mode 100644 index 0000000..1f086bb --- /dev/null +++ b/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs @@ -0,0 +1,96 @@ +module FsCodec.SystemTextJson.Tests.CompressionTests + +open Swensen.Unquote +open System +open System.Text.Json +open Xunit + +let inline roundtrip (sut: FsCodec.IEventCodec<'event, 'F, unit>) value = + let encoded = sut.Encode((), value = value) + let loaded = FsCodec.Core.TimelineEvent.Create(-1L, encoded) + sut.Decode loaded + +(* Base Fixture Round-trips a String encoded as JsonElement *) + +module StringUtf8 = + + let eventType = "EventType" + let enc (x: 't): JsonElement = JsonSerializer.SerializeToElement x + let dec (b: JsonElement): 't = JsonSerializer.Deserialize b + let jsonElementCodec<'t> = + let encode e = struct (eventType, enc e) + let decode s (b: JsonElement) = if s = eventType then ValueSome (dec b) else invalidOp "Invalid eventType value" + FsCodec.Codec.Create(encode, decode) + + let sut<'t> = jsonElementCodec<'t> + + let [] roundtrips () = + let value = {| value = "Hello World" |} + let res' = roundtrip sut value + res' =! ValueSome value + +module InternalDecoding = + + let inputValue = {| value = "Hello World" |} + // A JsonElement that's a JSON Object should be handled as an uncompressed value + let direct = struct (0, JsonSerializer.SerializeToElement inputValue) + // A JsonElement that's a JSON String should be treated as base64'd Deflate data where the Decoding is unspecified + let implicitDeflate = struct (Unchecked.defaultof, JsonSerializer.SerializeToElement "qlYqS8wpTVWyUvJIzcnJVwjPL8pJUaoFAAAA//8=") + let explicitDeflate = struct (1, JsonSerializer.SerializeToElement "qlYqS8wpTVWyUvJIzcnJVwjPL8pJUaoFAAAA//8=") + let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") + + let decode useRom = + if useRom then FsCodec.SystemTextJson.Compression.DecodeToByteArray >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.Compression.DecodeToJsonElement >> JsonSerializer.Deserialize + let [] ``Can decode all known representations`` useRom = + test <@ decode useRom direct = inputValue @> + test <@ decode useRom implicitDeflate = inputValue @> + test <@ decode useRom explicitDeflate = inputValue @> + test <@ decode useRom explicitBrotli = inputValue @> + + let [] ``Defaults to leaving the body alone if unknown`` useRom = + let struct (_, je) = direct + let body = struct (99, je) + let decoded = decode useRom body + test <@ decoded = inputValue @> + +type JsonElement with member x.Utf8ByteCount = if x.ValueKind = JsonValueKind.Null then 0 else x.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount + +module TryCompress = + + let sut = FsCodec.SystemTextJson.Compression.EncodeTryCompress StringUtf8.sut + + let compressibleValue = {| value = String('x', 5000) |} + + let [] roundtrips () = + let res' = roundtrip sut compressibleValue + res' =! ValueSome compressibleValue + + let [] ``compresses when possible`` () = + let encoded = sut.Encode((), value = compressibleValue) + let struct (_encoding, encodedValue) = encoded.Data + encodedValue.Utf8ByteCount ] ``produces equivalent JsonElement where compression not possible`` () = + let value = {| value = "NotCompressible" |} + let directResult = StringUtf8.sut.Encode((), value).Data + let failedToCompressResult = sut.Encode((), value = value) + let struct (_encoding, result) = failedToCompressResult.Data + true =! JsonElement.DeepEquals(directResult, result) + +module Uncompressed = + + let sut = FsCodec.SystemTextJson.Compression.EncodeUncompressed StringUtf8.sut + + // Borrow the value we just demonstrated to be compressible + let compressibleValue = TryCompress.compressibleValue + + let [] roundtrips () = + let res' = roundtrip sut compressibleValue + res' =! ValueSome compressibleValue + + let [] ``does not compress (despite it being possible to)`` () = + let directResult = StringUtf8.sut.Encode((), compressibleValue).Data + let shouldNotBeCompressedResult = sut.Encode((), value = compressibleValue) + let struct (_encoding, result) = shouldNotBeCompressedResult.Data + result.Utf8ByteCount =! directResult.Utf8ByteCount diff --git a/tests/FsCodec.SystemTextJson.Tests/Examples.fsx b/tests/FsCodec.SystemTextJson.Tests/Examples.fsx index ee24363..3f40945 100755 --- a/tests/FsCodec.SystemTextJson.Tests/Examples.fsx +++ b/tests/FsCodec.SystemTextJson.Tests/Examples.fsx @@ -511,3 +511,26 @@ Client ClientB, event 2 meta { principal = "me" } event Removed { name = null } Unhandled Event: Category Misc, Id x, Index 0, Event: "Dummy" *) + +(* Well known states for Compression regression tests *) + +open System +let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = + let output = new System.IO.MemoryStream() + use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + compressor.Write eventBody.Span + compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing + output + +/// Equinox.Cosmos / Equinox.CosmosStore Deflate logic was exactly as below, do not tweak: +let private deflate (uncompressedBytes: byte[]) = + let output = new System.IO.MemoryStream() + let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + compressor.Write(uncompressedBytes) + compressor.Flush() // Could `Close`, but not required + output.ToArray() +let raw = {| value = "Hello World" |} + +[| raw |> System.Text.Json.JsonSerializer.SerializeToUtf8Bytes |> ReadOnlyMemory |> brotliCompress |> _.ToArray() + raw |> System.Text.Json.JsonSerializer.SerializeToUtf8Bytes |> deflate |] +|> Array.map Convert.ToBase64String diff --git a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj index ac6b204..43d5c82 100644 --- a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj +++ b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj @@ -43,6 +43,7 @@ SomeNullHandlingTests.fs + From 87cba4df9a7bae40ed893e915d51c6ba58b7e040 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 Jan 2025 13:35:50 +0000 Subject: [PATCH 02/11] Polish/updates --- src/FsCodec.SystemTextJson/Compression.fs | 57 +++++++++---------- .../FsCodec.SystemTextJson.fsproj | 2 +- .../CompressionTests.fs | 4 +- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/FsCodec.SystemTextJson/Compression.fs b/src/FsCodec.SystemTextJson/Compression.fs index d0a12fe..adf8b2a 100644 --- a/src/FsCodec.SystemTextJson/Compression.fs +++ b/src/FsCodec.SystemTextJson/Compression.fs @@ -27,28 +27,23 @@ module private EncodedMaybeCompressed = // compressor.Write(uncompressedBytes) // compressor.Flush() // Could `Close`, but not required // output.ToArray() - let private inflate (compressedBytes: byte[]) = + let private inflateTo output (compressedBytes: byte[]) = let input = new System.IO.MemoryStream(compressedBytes) let decompressor = new System.IO.Compression.DeflateStream(input, System.IO.Compression.CompressionMode.Decompress, leaveOpen = true) - let output = new System.IO.MemoryStream() decompressor.CopyTo output - output.ToArray() - let private brotliDecompress (data: byte[]): byte[] = + let private brotliDecompressTo output (data: byte[]) = let s = new System.IO.MemoryStream(data, writable = false) use decompressor = new System.IO.Compression.BrotliStream(s, System.IO.Compression.CompressionMode.Decompress) - use output = new System.IO.MemoryStream() decompressor.CopyTo output - output.ToArray() - let decodeJsonElement struct (encoding, data: JsonElement): JsonElement = - match encoding, data.ValueKind with - | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> inflate |> InteropHelpers.Utf8ToJsonElement - | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> brotliDecompress |> InteropHelpers.Utf8ToJsonElement - | _ -> data - let decodeUtf8 struct (encoding, data: JsonElement): ReadOnlyMemory = + let expand post alg compressedBytes = + use output = new System.IO.MemoryStream() + compressedBytes |> alg output + output.ToArray() |> post + let decode direct expand struct (encoding, data: JsonElement) = match encoding, data.ValueKind with - | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> inflate |> ReadOnlyMemory - | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> brotliDecompress |> ReadOnlyMemory - | _ -> InteropHelpers.JsonElementToUtf8 data + | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> expand inflateTo + | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo + | _ -> data |> direct (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) @@ -87,20 +82,22 @@ type [] CompressionOptions = { minSize: int; minGain: int } with [] type Compression private () = - static member Encode(x: JsonElement): EncodedBody = + static member Direct(x: JsonElement): EncodedBody = EncodedMaybeCompressed.encodeUncompressed x - static member Encode(x: ReadOnlyMemory): EncodedBody = + static member Direct(x: ReadOnlyMemory): EncodedBody = EncodedMaybeCompressed.encodeUncompressedUtf8 x - static member EncodeTryCompress(options, x: JsonElement): EncodedBody = + static member MaybeCompress(options, x: JsonElement): EncodedBody = EncodedMaybeCompressed.tryCompress options.minSize options.minGain x - static member EncodeTryCompress(options, x: ReadOnlyMemory): EncodedBody = + static member MaybeCompress(options, x: ReadOnlyMemory): EncodedBody = EncodedMaybeCompressed.tryCompressUtf8 options.minSize options.minGain x - static member DecodeToJsonElement(x: EncodedBody): JsonElement = - EncodedMaybeCompressed.decodeJsonElement x - static member DecodeToUtf8(x: EncodedBody): ReadOnlyMemory = - EncodedMaybeCompressed.decodeUtf8 x - static member DecodeToByteArray(x: EncodedBody): byte[] = - Compression.DecodeToUtf8(x).ToArray() + static member ToJsonElement(x: EncodedBody): JsonElement = + EncodedMaybeCompressed.decode id (EncodedMaybeCompressed.expand InteropHelpers.Utf8ToJsonElement) x + static member ToUtf8(x: EncodedBody): ReadOnlyMemory = + EncodedMaybeCompressed.decode InteropHelpers.JsonElementToUtf8 (EncodedMaybeCompressed.expand ReadOnlyMemory) x + static member ToByteArray(x: EncodedBody): byte[] = + Compression.ToUtf8(x).ToArray() + static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = + EncodedMaybeCompressed.decode (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -109,7 +106,7 @@ type Compression private () = static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.EncodeTryCompress(opts, x)), Func<_, _> Compression.DecodeToUtf8) + FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.MaybeCompress(opts, x)), Func<_, _> Compression.ToUtf8) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -118,22 +115,22 @@ type Compression private () = static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.EncodeTryCompress(opts, x)), Func<_, _> Compression.DecodeToJsonElement) + FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.MaybeCompress(opts, x)), Func<_, _> Compression.ToJsonElement) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Encode, Func<_, _> Compression.DecodeToJsonElement) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Direct, Func<_, _> Compression.ToJsonElement) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.DecodeToUtf8, Func<_, _> Compression.Encode) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.ToUtf8, Func<_, _> Compression.Direct) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.DecodeToByteArray, Func<_, _> Compression.Encode) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.ToByteArray, Func<_, _> Compression.Direct) diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index 7c55368..07f0213 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -18,8 +18,8 @@ - + diff --git a/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs b/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs index 1f086bb..a35171e 100644 --- a/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs @@ -40,8 +40,8 @@ module InternalDecoding = let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.Compression.DecodeToByteArray >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.Compression.DecodeToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.Compression.ToByteArray >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.Compression.ToJsonElement >> JsonSerializer.Deserialize let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> test <@ decode useRom implicitDeflate = inputValue @> From 4c27ee833b9db5420665f3aa6f33414a7d9dad6e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 Jan 2025 17:43:09 +0000 Subject: [PATCH 03/11] Remove assumptions, finalize naming --- .../{Compression.fs => EncodedBody.fs} | 46 ++++++++++--------- .../FsCodec.SystemTextJson.fsproj | 2 +- ...ompressionTests.fs => EncodedBodyTests.fs} | 19 ++++---- .../FsCodec.SystemTextJson.Tests.fsproj | 2 +- 4 files changed, 37 insertions(+), 32 deletions(-) rename src/FsCodec.SystemTextJson/{Compression.fs => EncodedBody.fs} (80%) rename tests/FsCodec.SystemTextJson.Tests/{CompressionTests.fs => EncodedBodyTests.fs} (83%) diff --git a/src/FsCodec.SystemTextJson/Compression.fs b/src/FsCodec.SystemTextJson/EncodedBody.fs similarity index 80% rename from src/FsCodec.SystemTextJson/Compression.fs rename to src/FsCodec.SystemTextJson/EncodedBody.fs index adf8b2a..74edcc9 100644 --- a/src/FsCodec.SystemTextJson/Compression.fs +++ b/src/FsCodec.SystemTextJson/EncodedBody.fs @@ -11,7 +11,7 @@ open System.Text.Json /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used. type EncodedBody = (struct(int * JsonElement)) -module private EncodedMaybeCompressed = +module private Compression = module Encoding = let [] Direct = 0 // Assumed for all values not listed here @@ -39,11 +39,13 @@ module private EncodedMaybeCompressed = use output = new System.IO.MemoryStream() compressedBytes |> alg output output.ToArray() |> post - let decode direct expand struct (encoding, data: JsonElement) = + let decode_ direct expand struct (encoding, data: JsonElement) = match encoding, data.ValueKind with - | (Encoding.Direct | Encoding.Deflate), JsonValueKind.String -> data.GetBytesFromBase64() |> expand inflateTo + | Encoding.Deflate, JsonValueKind.String -> data.GetBytesFromBase64() |> expand inflateTo | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo | _ -> data |> direct + let decode = decode_ id (expand InteropHelpers.Utf8ToJsonElement) + let decodeUtf8 = decode_ InteropHelpers.JsonElementToUtf8 (expand ReadOnlyMemory) (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) @@ -80,24 +82,24 @@ type [] CompressionOptions = { minSize: int; minGain: int } with static member Default = { minSize = 48; minGain = 4 } [] -type Compression private () = - - static member Direct(x: JsonElement): EncodedBody = - EncodedMaybeCompressed.encodeUncompressed x - static member Direct(x: ReadOnlyMemory): EncodedBody = - EncodedMaybeCompressed.encodeUncompressedUtf8 x - static member MaybeCompress(options, x: JsonElement): EncodedBody = - EncodedMaybeCompressed.tryCompress options.minSize options.minGain x - static member MaybeCompress(options, x: ReadOnlyMemory): EncodedBody = - EncodedMaybeCompressed.tryCompressUtf8 options.minSize options.minGain x +type EncodedBodyExtensions private () = + + static member Uncompressed(x: JsonElement): EncodedBody = + Compression.encodeUncompressed x + static member Uncompressed(x: ReadOnlyMemory): EncodedBody = + Compression.encodeUncompressedUtf8 x + static member TryCompress(options, x: JsonElement): EncodedBody = + Compression.tryCompress options.minSize options.minGain x + static member TryCompress(options, x: ReadOnlyMemory): EncodedBody = + Compression.tryCompressUtf8 options.minSize options.minGain x static member ToJsonElement(x: EncodedBody): JsonElement = - EncodedMaybeCompressed.decode id (EncodedMaybeCompressed.expand InteropHelpers.Utf8ToJsonElement) x + Compression.decode x static member ToUtf8(x: EncodedBody): ReadOnlyMemory = - EncodedMaybeCompressed.decode InteropHelpers.JsonElementToUtf8 (EncodedMaybeCompressed.expand ReadOnlyMemory) x + Compression.decodeUtf8 x static member ToByteArray(x: EncodedBody): byte[] = - Compression.ToUtf8(x).ToArray() + EncodedBodyExtensions.ToUtf8(x).ToArray() static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = - EncodedMaybeCompressed.decode (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x + Compression.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -106,7 +108,7 @@ type Compression private () = static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.MaybeCompress(opts, x)), Func<_, _> Compression.ToUtf8) + FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBodyExtensions.TryCompress(opts, x)), Func<_, _> EncodedBodyExtensions.ToUtf8) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -115,22 +117,22 @@ type Compression private () = static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> Compression.MaybeCompress(opts, x)), Func<_, _> Compression.ToJsonElement) + FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBodyExtensions.TryCompress(opts, x)), Func<_, _> EncodedBodyExtensions.ToJsonElement) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Direct, Func<_, _> Compression.ToJsonElement) + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.Uncompressed, Func<_, _> EncodedBodyExtensions.ToJsonElement) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.ToUtf8, Func<_, _> Compression.Direct) + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.ToUtf8, Func<_, _> EncodedBodyExtensions.Uncompressed) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.ToByteArray, Func<_, _> Compression.Direct) + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.ToByteArray, Func<_, _> EncodedBodyExtensions.Uncompressed) diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index 07f0213..28f4408 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -19,7 +19,7 @@ - + diff --git a/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs b/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs similarity index 83% rename from tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs rename to tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs index a35171e..ccc6e1b 100644 --- a/tests/FsCodec.SystemTextJson.Tests/CompressionTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs @@ -1,4 +1,4 @@ -module FsCodec.SystemTextJson.Tests.CompressionTests +module FsCodec.SystemTextJson.Tests.EncodedBodyTests open Swensen.Unquote open System @@ -34,17 +34,15 @@ module InternalDecoding = let inputValue = {| value = "Hello World" |} // A JsonElement that's a JSON Object should be handled as an uncompressed value let direct = struct (0, JsonSerializer.SerializeToElement inputValue) - // A JsonElement that's a JSON String should be treated as base64'd Deflate data where the Decoding is unspecified - let implicitDeflate = struct (Unchecked.defaultof, JsonSerializer.SerializeToElement "qlYqS8wpTVWyUvJIzcnJVwjPL8pJUaoFAAAA//8=") let explicitDeflate = struct (1, JsonSerializer.SerializeToElement "qlYqS8wpTVWyUvJIzcnJVwjPL8pJUaoFAAAA//8=") let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.Compression.ToByteArray >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.Compression.ToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.EncodedBodyExtensions.ToByteArray >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.EncodedBodyExtensions.ToJsonElement >> JsonSerializer.Deserialize + let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> - test <@ decode useRom implicitDeflate = inputValue @> test <@ decode useRom explicitDeflate = inputValue @> test <@ decode useRom explicitBrotli = inputValue @> @@ -54,11 +52,16 @@ module InternalDecoding = let decoded = decode useRom body test <@ decoded = inputValue @> + let [] ``Defaults to leaving the body alone if string`` useRom = + let body = struct (99, JsonSerializer.SerializeToElement "test") + let decoded = decode useRom body + test <@ "test" = decoded @> + type JsonElement with member x.Utf8ByteCount = if x.ValueKind = JsonValueKind.Null then 0 else x.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount module TryCompress = - let sut = FsCodec.SystemTextJson.Compression.EncodeTryCompress StringUtf8.sut + let sut = FsCodec.SystemTextJson.EncodedBodyExtensions.EncodeTryCompress StringUtf8.sut let compressibleValue = {| value = String('x', 5000) |} @@ -80,7 +83,7 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.SystemTextJson.Compression.EncodeUncompressed StringUtf8.sut + let sut = FsCodec.SystemTextJson.EncodedBodyExtensions.EncodeUncompressed StringUtf8.sut // Borrow the value we just demonstrated to be compressible let compressibleValue = TryCompress.compressibleValue diff --git a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj index 43d5c82..ae7c372 100644 --- a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj +++ b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj @@ -43,7 +43,7 @@ SomeNullHandlingTests.fs - + From f492c94d37d43f126d18a497f01b221b1889a062 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 Jan 2025 18:39:02 +0000 Subject: [PATCH 04/11] Final name? --- src/FsCodec.SystemTextJson/EncodedBody.fs | 68 +++++++++---------- .../EncodedBodyTests.fs | 8 +-- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/FsCodec.SystemTextJson/EncodedBody.fs b/src/FsCodec.SystemTextJson/EncodedBody.fs index 74edcc9..71c1f7e 100644 --- a/src/FsCodec.SystemTextJson/EncodedBody.fs +++ b/src/FsCodec.SystemTextJson/EncodedBody.fs @@ -9,9 +9,9 @@ open System.Text.Json /// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value identifying the encoding scheme. /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used. -type EncodedBody = (struct(int * JsonElement)) +type EncodedBodyT = (struct(int * JsonElement)) -module private Compression = +module private Impl = module Encoding = let [] Direct = 0 // Assumed for all values not listed here @@ -50,7 +50,7 @@ module private Compression = (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) - let encodeUncompressed (raw: JsonElement): EncodedBody = Encoding.Direct, raw + let encodeUncompressed (raw: JsonElement): EncodedBodyT = Encoding.Direct, raw let private blobToStringElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = let output = new System.IO.MemoryStream() @@ -58,15 +58,15 @@ module private Compression = compressor.Write eventBody.Span compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing output - let tryCompress minSize minGain (raw: JsonElement): EncodedBody = + let tryCompress minSize minGain (raw: JsonElement): EncodedBodyT = let utf8: ReadOnlyMemory = InteropHelpers.JsonElementToUtf8 raw if utf8.Length < minSize then encodeUncompressed raw else let brotli = brotliCompress utf8 if utf8.Length <= int brotli.Length + minGain then encodeUncompressed raw else Encoding.Brotli, brotli.ToArray() |> blobToStringElement - let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw - let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = + let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBodyT = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw + let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBodyT = if utf8.Length < minSize then encodeUncompressedUtf8 utf8 else let brotli = brotliCompress utf8 @@ -82,57 +82,57 @@ type [] CompressionOptions = { minSize: int; minGain: int } with static member Default = { minSize = 48; minGain = 4 } [] -type EncodedBodyExtensions private () = - - static member Uncompressed(x: JsonElement): EncodedBody = - Compression.encodeUncompressed x - static member Uncompressed(x: ReadOnlyMemory): EncodedBody = - Compression.encodeUncompressedUtf8 x - static member TryCompress(options, x: JsonElement): EncodedBody = - Compression.tryCompress options.minSize options.minGain x - static member TryCompress(options, x: ReadOnlyMemory): EncodedBody = - Compression.tryCompressUtf8 options.minSize options.minGain x - static member ToJsonElement(x: EncodedBody): JsonElement = - Compression.decode x - static member ToUtf8(x: EncodedBody): ReadOnlyMemory = - Compression.decodeUtf8 x - static member ToByteArray(x: EncodedBody): byte[] = - EncodedBodyExtensions.ToUtf8(x).ToArray() - static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = - Compression.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x +type EncodedBody private () = + + static member Uncompressed(x: JsonElement): EncodedBodyT = + Impl.encodeUncompressed x + static member Uncompressed(x: ReadOnlyMemory): EncodedBodyT = + Impl.encodeUncompressedUtf8 x + static member TryCompress(options, x: JsonElement): EncodedBodyT = + Impl.tryCompress options.minSize options.minGain x + static member TryCompress(options, x: ReadOnlyMemory): EncodedBodyT = + Impl.tryCompressUtf8 options.minSize options.minGain x + static member ToJsonElement(x: EncodedBodyT): JsonElement = + Impl.decode x + static member ToUtf8(x: EncodedBodyT): ReadOnlyMemory = + Impl.decodeUtf8 x + static member ToByteArray(x: EncodedBodyT): byte[] = + EncodedBody.ToUtf8(x).ToArray() + static member ExpandTo(ms: System.IO.Stream, x: EncodedBodyT) = + Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. [] static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, EncodedBodyT, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBodyExtensions.TryCompress(opts, x)), Func<_, _> EncodedBodyExtensions.ToUtf8) + FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBody.TryCompress(opts, x)), Func<_, _> EncodedBody.ToUtf8) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. [] static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, EncodedBodyT, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBodyExtensions.TryCompress(opts, x)), Func<_, _> EncodedBodyExtensions.ToJsonElement) + FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBody.TryCompress(opts, x)), Func<_, _> EncodedBody.ToJsonElement) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) - : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.Uncompressed, Func<_, _> EncodedBodyExtensions.ToJsonElement) + : IEventCodec<'Event, EncodedBodyT, 'Context> = + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.Uncompressed, Func<_, _> EncodedBody.ToJsonElement) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] - static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBodyT, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.ToUtf8, Func<_, _> EncodedBodyExtensions.Uncompressed) + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.ToUtf8, Func<_, _> EncodedBody.Uncompressed) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] - static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBodyT, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBodyExtensions.ToByteArray, Func<_, _> EncodedBodyExtensions.Uncompressed) + FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.ToByteArray, Func<_, _> EncodedBody.Uncompressed) diff --git a/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs b/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs index ccc6e1b..3a894f7 100644 --- a/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs @@ -38,8 +38,8 @@ module InternalDecoding = let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.EncodedBodyExtensions.ToByteArray >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.EncodedBodyExtensions.ToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.EncodedBody.ToByteArray >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.EncodedBody.ToJsonElement >> JsonSerializer.Deserialize let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> @@ -61,7 +61,7 @@ type JsonElement with member x.Utf8ByteCount = if x.ValueKind = JsonValueKind.Nu module TryCompress = - let sut = FsCodec.SystemTextJson.EncodedBodyExtensions.EncodeTryCompress StringUtf8.sut + let sut = FsCodec.SystemTextJson.EncodedBody.EncodeTryCompress StringUtf8.sut let compressibleValue = {| value = String('x', 5000) |} @@ -83,7 +83,7 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.SystemTextJson.EncodedBodyExtensions.EncodeUncompressed StringUtf8.sut + let sut = FsCodec.SystemTextJson.EncodedBody.EncodeUncompressed StringUtf8.sut // Borrow the value we just demonstrated to be compressible let compressibleValue = TryCompress.compressibleValue From 63fb8701bb0f84d5551b8fe2043c9aed0028ca99 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 2 Jan 2025 00:19:06 +0000 Subject: [PATCH 05/11] Add shouldCompress predicate --- .../{EncodedBody.fs => Encoding.fs} | 91 +++++++++++-------- .../FsCodec.SystemTextJson.fsproj | 2 +- .../{EncodedBodyTests.fs => EncodingTests.fs} | 10 +- .../FsCodec.SystemTextJson.Tests.fsproj | 2 +- 4 files changed, 60 insertions(+), 45 deletions(-) rename src/FsCodec.SystemTextJson/{EncodedBody.fs => Encoding.fs} (61%) rename tests/FsCodec.SystemTextJson.Tests/{EncodedBodyTests.fs => EncodingTests.fs} (90%) diff --git a/src/FsCodec.SystemTextJson/EncodedBody.fs b/src/FsCodec.SystemTextJson/Encoding.fs similarity index 61% rename from src/FsCodec.SystemTextJson/EncodedBody.fs rename to src/FsCodec.SystemTextJson/Encoding.fs index 71c1f7e..99af8cb 100644 --- a/src/FsCodec.SystemTextJson/EncodedBody.fs +++ b/src/FsCodec.SystemTextJson/Encoding.fs @@ -9,7 +9,7 @@ open System.Text.Json /// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value identifying the encoding scheme. /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used. -type EncodedBodyT = (struct(int * JsonElement)) +type EncodedBody = (struct(int * JsonElement)) module private Impl = @@ -35,7 +35,7 @@ module private Impl = let s = new System.IO.MemoryStream(data, writable = false) use decompressor = new System.IO.Compression.BrotliStream(s, System.IO.Compression.CompressionMode.Decompress) decompressor.CopyTo output - let expand post alg compressedBytes = + let private unpack post alg compressedBytes = use output = new System.IO.MemoryStream() compressedBytes |> alg output output.ToArray() |> post @@ -44,34 +44,34 @@ module private Impl = | Encoding.Deflate, JsonValueKind.String -> data.GetBytesFromBase64() |> expand inflateTo | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo | _ -> data |> direct - let decode = decode_ id (expand InteropHelpers.Utf8ToJsonElement) - let decodeUtf8 = decode_ InteropHelpers.JsonElementToUtf8 (expand ReadOnlyMemory) + let decode = decode_ id (unpack InteropHelpers.Utf8ToJsonElement) + let decodeUtf8 = decode_ InteropHelpers.JsonElementToUtf8 (unpack ReadOnlyMemory) (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) - let encodeUncompressed (raw: JsonElement): EncodedBodyT = Encoding.Direct, raw - let private blobToStringElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement + let encodeUncompressed (raw: JsonElement): EncodedBody = Encoding.Direct, raw + let private blobToBase64StringJsonElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = let output = new System.IO.MemoryStream() use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) compressor.Write eventBody.Span compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing output - let tryCompress minSize minGain (raw: JsonElement): EncodedBodyT = + let tryCompress minSize minGain (raw: JsonElement): EncodedBody = let utf8: ReadOnlyMemory = InteropHelpers.JsonElementToUtf8 raw if utf8.Length < minSize then encodeUncompressed raw else let brotli = brotliCompress utf8 if utf8.Length <= int brotli.Length + minGain then encodeUncompressed raw else - Encoding.Brotli, brotli.ToArray() |> blobToStringElement - let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBodyT = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw - let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBodyT = + Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement + let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw + let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = if utf8.Length < minSize then encodeUncompressedUtf8 utf8 else let brotli = brotliCompress utf8 if utf8.Length <= int brotli.Length + minGain then encodeUncompressedUtf8 utf8 else - Encoding.Brotli, brotli.ToArray() |> blobToStringElement + Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement type [] CompressionOptions = { minSize: int; minGain: int } with /// Attempt to compress anything possible @@ -82,57 +82,72 @@ type [] CompressionOptions = { minSize: int; minGain: int } with static member Default = { minSize = 48; minGain = 4 } [] -type EncodedBody private () = +type Encoding private () = - static member Uncompressed(x: JsonElement): EncodedBodyT = + static member Uncompressed(x: JsonElement): EncodedBody = Impl.encodeUncompressed x - static member Uncompressed(x: ReadOnlyMemory): EncodedBodyT = + static member Uncompressed(x: ReadOnlyMemory): EncodedBody = Impl.encodeUncompressedUtf8 x - static member TryCompress(options, x: JsonElement): EncodedBodyT = + static member TryCompress(options, x: JsonElement): EncodedBody = Impl.tryCompress options.minSize options.minGain x - static member TryCompress(options, x: ReadOnlyMemory): EncodedBodyT = + static member TryCompress(options, x: ReadOnlyMemory): EncodedBody = Impl.tryCompressUtf8 options.minSize options.minGain x - static member ToJsonElement(x: EncodedBodyT): JsonElement = + static member ToJsonElement(x: EncodedBody): JsonElement = Impl.decode x - static member ToUtf8(x: EncodedBodyT): ReadOnlyMemory = + static member ToUtf8(x: EncodedBody): ReadOnlyMemory = Impl.decodeUtf8 x - static member ToByteArray(x: EncodedBodyT): byte[] = - EncodedBody.ToUtf8(x).ToArray() - static member ExpandTo(ms: System.IO.Stream, x: EncodedBodyT) = + static member ToByteArray(x: EncodedBody): byte[] = + Encoding.ToUtf8(x).ToArray() + static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x - /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
- /// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
- /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. + /// The body will be saved as-is under the following circumstances:
+ /// - the shouldCompress predicate is not satisfied for the event in question.
+ /// - sufficient compression, as defined by options is not achieved, the body is saved as-is.
+ /// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. [] - static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBodyT, 'Context> = + static member EncodeTryCompress<'Event, 'Context>( + native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, + [] ?shouldCompress: Func>, bool>, + [] ?options) + : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBody.TryCompress(opts, x)), Func<_, _> EncodedBody.ToUtf8) + let encode = shouldCompress |> function + | None -> fun _x (d: ReadOnlyMemory) -> Encoding.TryCompress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.Uncompressed d + FsCodec.Core.EventCodec.MapEx(native, encode, Func<_, _> Encoding.ToUtf8) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
- /// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
- /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. + /// The body will be saved as-is under the following circumstances:
+ /// - the shouldCompress predicate is not satisfied for the event in question.
+ /// - sufficient compression, as defined by options is not achieved, the body is saved as-is.
+ /// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. [] - static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBodyT, 'Context> = + static member EncodeTryCompress<'Event, 'Context>( + native: IEventCodec<'Event, JsonElement, 'Context>, + [] ?shouldCompress: Func, bool>, + [] ?options) + : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun x -> EncodedBody.TryCompress(opts, x)), Func<_, _> EncodedBody.ToJsonElement) + let encode = shouldCompress |> function + | None -> fun _x (d: JsonElement) -> Encoding.TryCompress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.Uncompressed d + FsCodec.Core.EventCodec.MapEx(native, encode, Func<_, _> Encoding.ToJsonElement) /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) - : IEventCodec<'Event, EncodedBodyT, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.Uncompressed, Func<_, _> EncodedBody.ToJsonElement) + : IEventCodec<'Event, EncodedBody, 'Context> = + FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.Uncompressed, Func<_, _> Encoding.ToJsonElement) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] - static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBodyT, 'Context>) + static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.ToUtf8, Func<_, _> EncodedBody.Uncompressed) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.ToUtf8, Func<_, _> Encoding.Uncompressed) /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] - static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBodyT, 'Context>) + static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> EncodedBody.ToByteArray, Func<_, _> EncodedBody.Uncompressed) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.ToByteArray, Func<_, _> Encoding.Uncompressed) diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index 28f4408..bb9f96d 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -19,7 +19,7 @@ - + diff --git a/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs similarity index 90% rename from tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs rename to tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs index 3a894f7..76002c2 100644 --- a/tests/FsCodec.SystemTextJson.Tests/EncodedBodyTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs @@ -1,4 +1,4 @@ -module FsCodec.SystemTextJson.Tests.EncodedBodyTests +module FsCodec.SystemTextJson.Tests.EncodingTests open Swensen.Unquote open System @@ -38,8 +38,8 @@ module InternalDecoding = let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.EncodedBody.ToByteArray >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.EncodedBody.ToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.Encoding.ToByteArray >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.Encoding.ToJsonElement >> JsonSerializer.Deserialize let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> @@ -61,7 +61,7 @@ type JsonElement with member x.Utf8ByteCount = if x.ValueKind = JsonValueKind.Nu module TryCompress = - let sut = FsCodec.SystemTextJson.EncodedBody.EncodeTryCompress StringUtf8.sut + let sut = FsCodec.SystemTextJson.Encoding.EncodeTryCompress StringUtf8.sut let compressibleValue = {| value = String('x', 5000) |} @@ -83,7 +83,7 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.SystemTextJson.EncodedBody.EncodeUncompressed StringUtf8.sut + let sut = FsCodec.SystemTextJson.Encoding.EncodeUncompressed StringUtf8.sut // Borrow the value we just demonstrated to be compressible let compressibleValue = TryCompress.compressibleValue diff --git a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj index ae7c372..dbc02fc 100644 --- a/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj +++ b/tests/FsCodec.SystemTextJson.Tests/FsCodec.SystemTextJson.Tests.fsproj @@ -43,7 +43,7 @@ SomeNullHandlingTests.fs - + From aa8dfb7bb3aee41a6bc93bdf7944c1c593fed054 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 2 Jan 2025 00:57:38 +0000 Subject: [PATCH 06/11] Ref updated FsCodec --- src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index bb9f96d..253e229 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -32,9 +32,10 @@ - - - + + + + From a4d8e1511ac43b47ceeed2bff565d743f32ca697 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 Jan 2025 10:30:02 +0000 Subject: [PATCH 07/11] Sync with MapBodies --- src/FsCodec.SystemTextJson/Encoding.fs | 35 ++++++++++--------- .../FsCodec.SystemTextJson.fsproj | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/FsCodec.SystemTextJson/Encoding.fs b/src/FsCodec.SystemTextJson/Encoding.fs index 99af8cb..7cebc5e 100644 --- a/src/FsCodec.SystemTextJson/Encoding.fs +++ b/src/FsCodec.SystemTextJson/Encoding.fs @@ -84,13 +84,13 @@ type [] CompressionOptions = { minSize: int; minGain: int } with [] type Encoding private () = - static member Uncompressed(x: JsonElement): EncodedBody = + static member FromJsonElement(x: JsonElement): EncodedBody = Impl.encodeUncompressed x - static member Uncompressed(x: ReadOnlyMemory): EncodedBody = + static member FromUtf8(x: ReadOnlyMemory): EncodedBody = Impl.encodeUncompressedUtf8 x static member TryCompress(options, x: JsonElement): EncodedBody = Impl.tryCompress options.minSize options.minGain x - static member TryCompress(options, x: ReadOnlyMemory): EncodedBody = + static member TryCompressUtf8(options, x: ReadOnlyMemory): EncodedBody = Impl.tryCompressUtf8 options.minSize options.minGain x static member ToJsonElement(x: EncodedBody): JsonElement = Impl.decode x @@ -101,21 +101,28 @@ type Encoding private () = static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x + /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. + [] + static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) + : IEventCodec<'Event, EncodedBody, 'Context> = + FsCodec.Core.EventCodec.mapBodies Encoding.FromJsonElement Encoding.ToJsonElement native + /// The body will be saved as-is under the following circumstances:
/// - the shouldCompress predicate is not satisfied for the event in question.
/// - sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. + /// NOTE this is intended for interoperability only; a Codec (such as CodecJsonElement) that encodes to JsonElement is strongly recommended unless you don't have a choice. [] - static member EncodeTryCompress<'Event, 'Context>( + static member EncodeTryCompressUtf8<'Event, 'Context>( native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?shouldCompress: Func>, bool>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function - | None -> fun _x (d: ReadOnlyMemory) -> Encoding.TryCompress(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.Uncompressed d - FsCodec.Core.EventCodec.MapEx(native, encode, Func<_, _> Encoding.ToUtf8) + | None -> fun _x (d: ReadOnlyMemory) -> Encoding.TryCompressUtf8(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompressUtf8(opts, d) else Encoding.FromUtf8 d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToUtf8 native /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// The body will be saved as-is under the following circumstances:
@@ -131,23 +138,17 @@ type Encoding private () = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function | None -> fun _x (d: JsonElement) -> Encoding.TryCompress(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.Uncompressed d - FsCodec.Core.EventCodec.MapEx(native, encode, Func<_, _> Encoding.ToJsonElement) - - /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. - [] - static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) - : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.Uncompressed, Func<_, _> Encoding.ToJsonElement) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.FromJsonElement d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToJsonElement native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.ToUtf8, Func<_, _> Encoding.Uncompressed) + FsCodec.Core.EventCodec.mapBodies Encoding.ToUtf8 Encoding.FromUtf8 native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Encoding.ToByteArray, Func<_, _> Encoding.Uncompressed) + FsCodec.Core.EventCodec.mapBodies Encoding.ToByteArray Encoding.FromUtf8 native diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index 253e229..25d1bb6 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -35,7 +35,7 @@ - + From 1b2cb49e90faa2febd36a092193271331908182b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 Jan 2025 23:01:45 +0000 Subject: [PATCH 08/11] Add base Encoding --- src/FsCodec.Box/ByteArray.fs | 2 +- src/FsCodec.Box/Compression.fs | 75 ++----------- src/FsCodec.Box/FsCodec.Box.fsproj | 6 +- src/FsCodec.SystemTextJson/Encoding.fs | 45 +++----- .../FsCodec.SystemTextJson.fsproj | 7 +- src/FsCodec.SystemTextJson/Interop.fs | 4 +- src/FsCodec/Encoding.fs | 104 ++++++++++++++++++ src/FsCodec/FsCodec.fs | 4 +- src/FsCodec/FsCodec.fsproj | 3 +- .../EncodingTests.fs | 4 +- .../{CompressionTests.fs => EncodingTests.fs} | 12 +- tests/FsCodec.Tests/FsCodec.Tests.fsproj | 3 +- 12 files changed, 152 insertions(+), 117 deletions(-) create mode 100644 src/FsCodec/Encoding.fs rename tests/FsCodec.Tests/{CompressionTests.fs => EncodingTests.fs} (87%) diff --git a/src/FsCodec.Box/ByteArray.fs b/src/FsCodec.Box/ByteArray.fs index 85c456f..77316d2 100644 --- a/src/FsCodec.Box/ByteArray.fs +++ b/src/FsCodec.Box/ByteArray.fs @@ -19,4 +19,4 @@ type ByteArray private () = [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> ByteArray.ReadOnlyMemoryToBytes, Func<_, _> ByteArray.BytesToReadOnlyMemory) + FsCodec.Core.EventCodec.mapBodies ByteArray.ReadOnlyMemoryToBytes ByteArray.BytesToReadOnlyMemory native diff --git a/src/FsCodec.Box/Compression.fs b/src/FsCodec.Box/Compression.fs index aaa7cf9..1b0622f 100644 --- a/src/FsCodec.Box/Compression.fs +++ b/src/FsCodec.Box/Compression.fs @@ -8,77 +8,21 @@ open System.Runtime.InteropServices /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used type EncodedBody = (struct(int * ReadOnlyMemory)) -module private EncodedMaybeCompressed = - - module Encoding = - let [] Direct = 0 // Assumed for all values not listed here - let [] Deflate = 1 // Deprecated encoding produced by versions pre 3.0.0-rc.13; no longer produced - let [] Brotli = 2 // Default encoding as of 3.0.0-rc.13 - - (* Decompression logic: triggered by extension methods below at the point where the Codec's Decode retrieves the Data or Meta properties *) - - // In versions pre 3.0.0-rc.13, the compression was implemented as follows; NOTE: use of Flush vs Close saves space but is unconventional - // let private deflate (eventBody: ReadOnlyMemory): System.IO.MemoryStream = - // let output = new System.IO.MemoryStream() - // let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) - // compressor.Write(eventBody.Span) - // compressor.Flush() // NOTE: using Flush in lieu of close means the result is not padded, which can hinder interop - // output - let private inflate (data: ReadOnlyMemory): byte[] = - let s = new System.IO.MemoryStream(data.ToArray(), writable = false) - let decompressor = new System.IO.Compression.DeflateStream(s, System.IO.Compression.CompressionMode.Decompress, leaveOpen = true) - let output = new System.IO.MemoryStream() - decompressor.CopyTo output - output.ToArray() - let private brotliDecompress (data: ReadOnlyMemory): byte[] = - let s = new System.IO.MemoryStream(data.ToArray(), writable = false) - use decompressor = new System.IO.Compression.BrotliStream(s, System.IO.Compression.CompressionMode.Decompress) - use output = new System.IO.MemoryStream() - decompressor.CopyTo output - output.ToArray() - let decode struct (encoding, data): ReadOnlyMemory = - match encoding with - | Encoding.Deflate -> inflate data |> ReadOnlyMemory - | Encoding.Brotli -> brotliDecompress data |> ReadOnlyMemory - | Encoding.Direct | _ -> data - - (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields - Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) - - let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = - let output = new System.IO.MemoryStream() - use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) - compressor.Write(eventBody.Span) - compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing - output - let encodeUncompressed (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, raw - let encode minSize minGain (raw: ReadOnlyMemory): EncodedBody = - if raw.Length < minSize then encodeUncompressed raw - else match brotliCompress raw with - | tmp when raw.Length > int tmp.Length + minGain -> Encoding.Brotli, tmp.ToArray() |> ReadOnlyMemory - | _ -> encodeUncompressed raw - type [] CompressionOptions = { minSize: int; minGain: int } with - /// Attempt to compress anything possible - // TL;DR in general it's worth compressing everything to minimize RU consumption both on insert and update - // For DynamoStore, every time we need to calve from the tip, the RU impact of using TransactWriteItems is significant, - // so preventing or delaying that is of critical importance - // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default static member Default = { minSize = 48; minGain = 4 } - /// Encode the data without attempting to compress, regardless of size static member Uncompressed = { minSize = Int32.MaxValue; minGain = 0 } -[] +[] type Compression private () = static member Utf8ToEncodedDirect(x: ReadOnlyMemory): EncodedBody = - EncodedMaybeCompressed.encodeUncompressed x + FsCodec.Encoding.FromBlob x static member Utf8ToEncodedTryCompress(options, x: ReadOnlyMemory): EncodedBody = - EncodedMaybeCompressed.encode options.minSize options.minGain x + FsCodec.Encoding.FromBlobTryCompress({ minSize = options.minSize; minGain = options.minGain }, x) static member EncodedToUtf8(x: EncodedBody): ReadOnlyMemory = - EncodedMaybeCompressed.decode x + FsCodec.Encoding.DecodeToBlob x static member EncodedToByteArray(x: EncodedBody): byte[] = - Compression.EncodedToUtf8(x).ToArray() + FsCodec.Encoding.DecodeToBlob(x).ToArray() /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -87,22 +31,23 @@ type Compression private () = static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, (fun d -> Compression.Utf8ToEncodedTryCompress(opts, d)), Func<_, _> Compression.EncodedToUtf8) + let opts: FsCodec.CompressionOptions = { minSize = opts.minSize; minGain = opts.minGain } + FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.FromBlobTryCompress(opts, d)) Encoding.DecodeToBlob native /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Utf8ToEncodedDirect, Func<_, _> Compression.EncodedToUtf8) + Encoding.EncodeUncompressed native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.EncodedToUtf8, Func<_, _> Compression.Utf8ToEncodedDirect) + Encoding.ToBlobCodec native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.EncodedToByteArray, Func<_, _> Compression.Utf8ToEncodedDirect) + Encoding.ToBlobArrayCodec native diff --git a/src/FsCodec.Box/FsCodec.Box.fsproj b/src/FsCodec.Box/FsCodec.Box.fsproj index 93ebe76..e56dd88 100644 --- a/src/FsCodec.Box/FsCodec.Box.fsproj +++ b/src/FsCodec.Box/FsCodec.Box.fsproj @@ -23,9 +23,9 @@ - - - + + + diff --git a/src/FsCodec.SystemTextJson/Encoding.fs b/src/FsCodec.SystemTextJson/Encoding.fs index 7cebc5e..98f09b5 100644 --- a/src/FsCodec.SystemTextJson/Encoding.fs +++ b/src/FsCodec.SystemTextJson/Encoding.fs @@ -13,11 +13,6 @@ type EncodedBody = (struct(int * JsonElement)) module private Impl = - module Encoding = - let [] Direct = 0 // Assumed for all values not listed here - let [] Deflate = 1 // Deprecated encoding produced by Equinox.Cosmos/CosmosStore < v 4.1.0; no longer produced - let [] Brotli = 2 // Default encoding - (* Decompression logic: triggered by extension methods below at the point where the Codec's Decode retrieves the Data or Meta properties *) // Equinox.Cosmos / Equinox.CosmosStore Deflate logic was as below: @@ -73,14 +68,6 @@ module private Impl = if utf8.Length <= int brotli.Length + minGain then encodeUncompressedUtf8 utf8 else Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement -type [] CompressionOptions = { minSize: int; minGain: int } with - /// Attempt to compress anything possible - // TL;DR in general it's worth compressing everything to minimize RU consumption both on insert and update - // For CosmosStore, every time we touch the tip, the RU impact of the write is significant, - // so preventing or delaying that is of critical importance - // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default - static member Default = { minSize = 48; minGain = 4 } - [] type Encoding private () = @@ -88,16 +75,14 @@ type Encoding private () = Impl.encodeUncompressed x static member FromUtf8(x: ReadOnlyMemory): EncodedBody = Impl.encodeUncompressedUtf8 x - static member TryCompress(options, x: JsonElement): EncodedBody = + static member FromJsonElementTryCompress(options, x: JsonElement): EncodedBody = Impl.tryCompress options.minSize options.minGain x - static member TryCompressUtf8(options, x: ReadOnlyMemory): EncodedBody = + static member FromUtf8TryCompress(options, x: ReadOnlyMemory): EncodedBody = Impl.tryCompressUtf8 options.minSize options.minGain x - static member ToJsonElement(x: EncodedBody): JsonElement = + static member DecodeToJsonElement(x: EncodedBody): JsonElement = Impl.decode x - static member ToUtf8(x: EncodedBody): ReadOnlyMemory = + static member DecodeToUtf8(x: EncodedBody): ReadOnlyMemory = Impl.decodeUtf8 x - static member ToByteArray(x: EncodedBody): byte[] = - Encoding.ToUtf8(x).ToArray() static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x @@ -105,7 +90,7 @@ type Encoding private () = [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.FromJsonElement Encoding.ToJsonElement native + FsCodec.Core.EventCodec.mapBodies Encoding.FromJsonElement Encoding.DecodeToJsonElement native /// The body will be saved as-is under the following circumstances:
/// - the shouldCompress predicate is not satisfied for the event in question.
@@ -113,16 +98,16 @@ type Encoding private () = /// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. /// NOTE this is intended for interoperability only; a Codec (such as CodecJsonElement) that encodes to JsonElement is strongly recommended unless you don't have a choice. [] - static member EncodeTryCompressUtf8<'Event, 'Context>( + static member EncodeUtf8TryCompress<'Event, 'Context>( native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?shouldCompress: Func>, bool>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function - | None -> fun _x (d: ReadOnlyMemory) -> Encoding.TryCompressUtf8(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompressUtf8(opts, d) else Encoding.FromUtf8 d - FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToUtf8 native + | None -> fun _x (d: ReadOnlyMemory) -> Encoding.FromUtf8TryCompress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.FromUtf8TryCompress(opts, d) else Encoding.FromUtf8 d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.DecodeToUtf8 native /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// The body will be saved as-is under the following circumstances:
@@ -137,18 +122,18 @@ type Encoding private () = : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function - | None -> fun _x (d: JsonElement) -> Encoding.TryCompress(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.TryCompress(opts, d) else Encoding.FromJsonElement d - FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToJsonElement native + | None -> fun _x (d: JsonElement) -> Encoding.FromJsonElementTryCompress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.FromJsonElementTryCompress(opts, d) else Encoding.FromJsonElement d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.DecodeToJsonElement native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.ToUtf8 Encoding.FromUtf8 native + FsCodec.Core.EventCodec.mapBodies Encoding.DecodeToUtf8 Encoding.FromUtf8 native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. [] - static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member ToUtf8ArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.ToByteArray Encoding.FromUtf8 native + FsCodec.Core.EventCodec.mapBodies (Encoding.DecodeToUtf8 >> _.ToArray()) Encoding.FromUtf8 native diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index 25d1bb6..ac7785f 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -32,10 +32,11 @@ - - + + + - + diff --git a/src/FsCodec.SystemTextJson/Interop.fs b/src/FsCodec.SystemTextJson/Interop.fs index 5e6bb1e..5230d1c 100644 --- a/src/FsCodec.SystemTextJson/Interop.fs +++ b/src/FsCodec.SystemTextJson/Interop.fs @@ -21,11 +21,11 @@ type InteropHelpers private () = [] static member ToUtf8Codec<'Event, 'Context>(native: FsCodec.IEventCodec<'Event, JsonElement, 'Context>) : FsCodec.IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> InteropHelpers.JsonElementToUtf8, Func<_, _> InteropHelpers.Utf8ToJsonElement) + FsCodec.Core.EventCodec.mapBodies InteropHelpers.JsonElementToUtf8 InteropHelpers.Utf8ToJsonElement native /// Adapts an IEventCodec that's rendering to ReadOnlyMemory<byte> Event Bodies to handle JsonElement bodies instead.
/// NOTE where possible, it's better to use CodecJsonElement in preference to Codec to encode directly in order to avoid this mapping process. [] static member ToJsonElementCodec<'Event, 'Context>(native: FsCodec.IEventCodec<'Event, ReadOnlyMemory, 'Context>) : FsCodec.IEventCodec<'Event, JsonElement, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> InteropHelpers.Utf8ToJsonElement, Func<_, _> InteropHelpers.JsonElementToUtf8) + FsCodec.Core.EventCodec.mapBodies InteropHelpers.Utf8ToJsonElement InteropHelpers.JsonElementToUtf8 native diff --git a/src/FsCodec/Encoding.fs b/src/FsCodec/Encoding.fs new file mode 100644 index 0000000..6f122f3 --- /dev/null +++ b/src/FsCodec/Encoding.fs @@ -0,0 +1,104 @@ +namespace FsCodec + +open System +open System.Runtime.CompilerServices +open System.Runtime.InteropServices + +/// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value signifying the encoding scheme. +/// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used +type EncodedBody = (struct(int * ReadOnlyMemory)) + +module Encoding = + let [] Direct = 0 // Assumed for all values not listed here + let [] Deflate = 1 // Deprecated encoding produced by versions pre 3.0.0-rc.13; no longer produced + let [] Brotli = 2 // Default encoding as of 3.0.0-rc.13 + +module private Impl = + + (* Decompression logic: triggered by extension methods below at the point where the Codec's Decode retrieves the Data or Meta properties *) + + // In versions pre 3.0.0-rc.13, the compression was implemented as follows; NOTE: use of Flush vs Close saves space but is unconventional + // let private deflate (eventBody: ReadOnlyMemory): System.IO.MemoryStream = + // let output = new System.IO.MemoryStream() + // let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + // compressor.Write(eventBody.Span) + // compressor.Flush() // NOTE: using Flush in lieu of close means the result is not padded, which can hinder interop + // output + let private inflateTo output (data: ReadOnlyMemory) = + let input = new System.IO.MemoryStream(data.ToArray(), writable = false) + let decompressor = new System.IO.Compression.DeflateStream(input, System.IO.Compression.CompressionMode.Decompress, leaveOpen = true) + decompressor.CopyTo output + let private brotliDecompressTo output (data: ReadOnlyMemory) = + let input = new System.IO.MemoryStream(data.ToArray(), writable = false) + use decompressor = new System.IO.Compression.BrotliStream(input, System.IO.Compression.CompressionMode.Decompress) + decompressor.CopyTo output + let private unpack alg compressedBytes = + use output = new System.IO.MemoryStream() + compressedBytes |> alg output + output.ToArray() |> ReadOnlyMemory + let decode struct (encoding, data): ReadOnlyMemory = + match encoding with + | Encoding.Deflate -> data |> unpack inflateTo + | Encoding.Brotli -> data |> unpack brotliDecompressTo + | Encoding.Direct | _ -> data + + (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields + Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) + + let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = + let output = new System.IO.MemoryStream() + use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + compressor.Write(eventBody.Span) + compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing + output + let encodeUncompressed (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, raw + let tryCompress minSize minGain (raw: ReadOnlyMemory): EncodedBody = + if raw.Length < minSize then encodeUncompressed raw + else match brotliCompress raw with + | tmp when raw.Length > int tmp.Length + minGain -> Encoding.Brotli, tmp.ToArray() |> ReadOnlyMemory + | _ -> encodeUncompressed raw + +type [] CompressionOptions = { minSize: int; minGain: int } with + /// Attempt to compress anything possible + // TL;DR in general it's worth compressing everything to minimize RU consumption both on insert and update + // For DynamoStore, every time we need to calve from the tip, the RU impact of using TransactWriteItems is significant, + // so preventing or delaying that is of critical importance + // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default + static member Default = { minSize = 48; minGain = 4 } + +[] +type Encoding private () = + + static member FromBlob(x: ReadOnlyMemory): EncodedBody = + Impl.encodeUncompressed x + static member FromBlobTryCompress(options, x: ReadOnlyMemory): EncodedBody = + Impl.tryCompress options.minSize options.minGain x + static member DecodeToBlob(x: EncodedBody): ReadOnlyMemory = + Impl.decode x + + /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to attempt to compress the data.
+ /// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
+ /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. + [] + static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) + : IEventCodec<'Event, EncodedBody, 'Context> = + let opts = defaultArg options CompressionOptions.Default + FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.FromBlobTryCompress(opts, d)) Encoding.DecodeToBlob native + + /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per EncodeTryCompress, but without attempting compression. + [] + static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) + : IEventCodec<'Event, EncodedBody, 'Context> = + FsCodec.Core.EventCodec.mapBodies Encoding.FromBlob Encoding.DecodeToBlob native + + /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. + [] + static member ToBlobCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + : IEventCodec<'Event, ReadOnlyMemory, 'Context> = + FsCodec.Core.EventCodec.mapBodies Encoding.DecodeToBlob Encoding.FromBlob native + + /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. + [] + static member ToBlobArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + : IEventCodec<'Event, byte[], 'Context> = + FsCodec.Core.EventCodec.mapBodies (Encoding.DecodeToBlob >> _.ToArray()) Encoding.FromBlob native diff --git a/src/FsCodec/FsCodec.fs b/src/FsCodec/FsCodec.fs index ff203ed..28311c8 100755 --- a/src/FsCodec/FsCodec.fs +++ b/src/FsCodec/FsCodec.fs @@ -169,9 +169,7 @@ type EventCodec<'Event, 'Format, 'Context> private () = let encoded = downConvert.Invoke target native.Decode encoded } - // NOTE To be be replaced by MapBodies/EventCodec.mapBodies for symmetry with TimelineEvent and EventData - // TO BE be Obsoleted and whenever FsCodec.Box is next released - [] + [] static member Map<'TargetFormat>(native: IEventCodec<'Event, 'Format, 'Context>, up: Func<'Format, 'TargetFormat>, down: Func<'TargetFormat, 'Format>) : IEventCodec<'Event, 'TargetFormat, 'Context> = EventCodec.MapBodies(native, Func<_, _, _>(fun _x -> up.Invoke), down) diff --git a/src/FsCodec/FsCodec.fsproj b/src/FsCodec/FsCodec.fsproj index 2fa424e..7cd10d0 100644 --- a/src/FsCodec/FsCodec.fsproj +++ b/src/FsCodec/FsCodec.fsproj @@ -3,13 +3,14 @@ - netstandard2.0 + netstandard2.1 3.0.0 + diff --git a/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs index 76002c2..9df7775 100644 --- a/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs @@ -38,8 +38,8 @@ module InternalDecoding = let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.Encoding.ToByteArray >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.Encoding.ToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.Encoding.DecodeToUtf8 >> _.ToArray() >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.Encoding.DecodeToJsonElement >> JsonSerializer.Deserialize let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> diff --git a/tests/FsCodec.Tests/CompressionTests.fs b/tests/FsCodec.Tests/EncodingTests.fs similarity index 87% rename from tests/FsCodec.Tests/CompressionTests.fs rename to tests/FsCodec.Tests/EncodingTests.fs index ab89e32..a73f1d8 100644 --- a/tests/FsCodec.Tests/CompressionTests.fs +++ b/tests/FsCodec.Tests/EncodingTests.fs @@ -1,4 +1,4 @@ -module FsCodec.Tests.CompressionTests +module FsCodec.Tests.EncodingTests open System open Swensen.Unquote @@ -30,7 +30,7 @@ module StringUtf8 = module TryCompress = - let sut = FsCodec.Compression.EncodeTryCompress(StringUtf8.sut) + let sut = FsCodec.Encoding.EncodeTryCompress(StringUtf8.sut) let compressibleValue = String('x', 5000) @@ -52,7 +52,7 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.Compression.EncodeUncompressed(StringUtf8.sut) + let sut = FsCodec.Encoding.EncodeUncompressed(StringUtf8.sut) // Borrow a demonstrably compressible value let value = TryCompress.compressibleValue @@ -74,13 +74,13 @@ module Decoding = let brotli = struct(2, Convert.FromBase64String("CwWASGVsbG8gV29ybGQ=") |> ReadOnlyMemory) let [] ``Can decode all known bodies`` () = - let decode = FsCodec.Compression.EncodedToByteArray >> Text.Encoding.UTF8.GetString + let decode = FsCodec.Encoding.DecodeToBlob >> _.ToArray() >> Text.Encoding.UTF8.GetString test <@ decode raw = "Hello World" @> test <@ decode deflated = "Hello World" @> test <@ decode brotli = "Hello World" @> let [] ``Defaults to leaving the memory alone if unknown`` () = let struct(_, mem) = raw - let body = struct(99, mem) - let decoded = body |> FsCodec.Compression.EncodedToByteArray |> Text.Encoding.UTF8.GetString + let body = struct (99, mem) + let decoded = body |> FsCodec.Encoding.DecodeToBlob |> _.ToArray() |> Text.Encoding.UTF8.GetString test <@ decoded = "Hello World" @> diff --git a/tests/FsCodec.Tests/FsCodec.Tests.fsproj b/tests/FsCodec.Tests/FsCodec.Tests.fsproj index e46994f..c635b36 100644 --- a/tests/FsCodec.Tests/FsCodec.Tests.fsproj +++ b/tests/FsCodec.Tests/FsCodec.Tests.fsproj @@ -7,7 +7,7 @@ - + @@ -21,6 +21,7 @@ + From 4a36bbf8c3aa0d65fa4c90ae522036e423c894ef Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 Jan 2025 23:19:25 +0000 Subject: [PATCH 09/11] Supress --- src/FsCodec/FsCodec.fsproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/FsCodec/FsCodec.fsproj b/src/FsCodec/FsCodec.fsproj index 7cd10d0..cb5b675 100644 --- a/src/FsCodec/FsCodec.fsproj +++ b/src/FsCodec/FsCodec.fsproj @@ -4,6 +4,8 @@ netstandard2.1 + + PKV006 3.0.0 From 52f35b8901a5e382879f7e9c8887a6a6c5cc743c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 Jan 2025 15:15:03 +0000 Subject: [PATCH 10/11] Encoder vs Encoding --- src/FsCodec.Box/Compression.fs | 16 ++-- src/FsCodec.SystemTextJson/Encoding.fs | 88 +++++++++++-------- src/FsCodec/Encoding.fs | 31 ++++--- .../EncodingTests.fs | 9 +- tests/FsCodec.Tests/EncodingTests.fs | 8 +- 5 files changed, 86 insertions(+), 66 deletions(-) diff --git a/src/FsCodec.Box/Compression.fs b/src/FsCodec.Box/Compression.fs index 1b0622f..0fc9d88 100644 --- a/src/FsCodec.Box/Compression.fs +++ b/src/FsCodec.Box/Compression.fs @@ -16,13 +16,13 @@ type [] CompressionOptions = { minSize: int; minGain: int } with type Compression private () = static member Utf8ToEncodedDirect(x: ReadOnlyMemory): EncodedBody = - FsCodec.Encoding.FromBlob x + FsCodec.Encoding.OfBlob x static member Utf8ToEncodedTryCompress(options, x: ReadOnlyMemory): EncodedBody = - FsCodec.Encoding.FromBlobTryCompress({ minSize = options.minSize; minGain = options.minGain }, x) + FsCodec.Encoding.OfBlobCompress({ minSize = options.minSize; minGain = options.minGain }, x) static member EncodedToUtf8(x: EncodedBody): ReadOnlyMemory = - FsCodec.Encoding.DecodeToBlob x + FsCodec.Encoding.ToBlob x static member EncodedToByteArray(x: EncodedBody): byte[] = - FsCodec.Encoding.DecodeToBlob(x).ToArray() + FsCodec.Encoding.ToBlob(x).ToArray() /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
@@ -32,22 +32,22 @@ type Compression private () = : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let opts: FsCodec.CompressionOptions = { minSize = opts.minSize; minGain = opts.minGain } - FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.FromBlobTryCompress(opts, d)) Encoding.DecodeToBlob native + FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.OfBlobCompress(opts, d)) Encoding.ToBlob native /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - Encoding.EncodeUncompressed native + Encoder.Uncompressed native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. [] static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - Encoding.ToBlobCodec native + Encoder.AsBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - Encoding.ToBlobArrayCodec native + Encoder.AsByteArray native diff --git a/src/FsCodec.SystemTextJson/Encoding.fs b/src/FsCodec.SystemTextJson/Encoding.fs index 98f09b5..5c63865 100644 --- a/src/FsCodec.SystemTextJson/Encoding.fs +++ b/src/FsCodec.SystemTextJson/Encoding.fs @@ -40,57 +40,71 @@ module private Impl = | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo | _ -> data |> direct let decode = decode_ id (unpack InteropHelpers.Utf8ToJsonElement) + let private blobToBase64StringJsonElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement + let direct (raw: JsonElement): EncodedBody = Encoding.Direct, raw + let recode struct (encoding, data: ReadOnlyMemory): EncodedBody = + match encoding with + | Encoding.Deflate -> Encoding.Deflate, data.ToArray() |> blobToBase64StringJsonElement + | Encoding.Brotli -> Encoding.Brotli, data.ToArray() |> blobToBase64StringJsonElement + | _ -> Encoding.Direct, data.ToArray() |> blobToBase64StringJsonElement let decodeUtf8 = decode_ InteropHelpers.JsonElementToUtf8 (unpack ReadOnlyMemory) (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) - let encodeUncompressed (raw: JsonElement): EncodedBody = Encoding.Direct, raw - let private blobToBase64StringJsonElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = let output = new System.IO.MemoryStream() use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) compressor.Write eventBody.Span compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing output - let tryCompress minSize minGain (raw: JsonElement): EncodedBody = + let compress minSize minGain (raw: JsonElement): EncodedBody = let utf8: ReadOnlyMemory = InteropHelpers.JsonElementToUtf8 raw - if utf8.Length < minSize then encodeUncompressed raw else + if utf8.Length < minSize then direct raw else let brotli = brotliCompress utf8 - if utf8.Length <= int brotli.Length + minGain then encodeUncompressed raw else + if utf8.Length <= int brotli.Length + minGain then direct raw else Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement - let encodeUncompressedUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw - let tryCompressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = - if utf8.Length < minSize then encodeUncompressedUtf8 utf8 else + let directUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw + let compressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = + if utf8.Length < minSize then directUtf8 utf8 else let brotli = brotliCompress utf8 - if utf8.Length <= int brotli.Length + minGain then encodeUncompressedUtf8 utf8 else + if utf8.Length <= int brotli.Length + minGain then directUtf8 utf8 else Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement -[] +[] type Encoding private () = - static member FromJsonElement(x: JsonElement): EncodedBody = - Impl.encodeUncompressed x - static member FromUtf8(x: ReadOnlyMemory): EncodedBody = - Impl.encodeUncompressedUtf8 x - static member FromJsonElementTryCompress(options, x: JsonElement): EncodedBody = - Impl.tryCompress options.minSize options.minGain x - static member FromUtf8TryCompress(options, x: ReadOnlyMemory): EncodedBody = - Impl.tryCompressUtf8 options.minSize options.minGain x - static member DecodeToJsonElement(x: EncodedBody): JsonElement = + static member OfJsonElement(x: JsonElement): EncodedBody = + Impl.direct x + static member OfJsonElementCompress(options, x: JsonElement): EncodedBody = + Impl.compress options.minSize options.minGain x + static member OfUtf8(x: ReadOnlyMemory): EncodedBody = + Impl.directUtf8 x + static member OfUtf8Compress(options, x: ReadOnlyMemory): EncodedBody = + Impl.compressUtf8 options.minSize options.minGain x + static member OfEncodedUtf8(x: FsCodec.EncodedBody): EncodedBody = + Impl.recode x + static member ByteCount((_encoding, data): EncodedBody) = + data.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount + static member ByteCountExpanded(x: EncodedBody) = + Impl.decode x |> _.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount + static member ToJsonElement(x: EncodedBody): JsonElement = Impl.decode x - static member DecodeToUtf8(x: EncodedBody): ReadOnlyMemory = + static member ToUtf8(x: EncodedBody): ReadOnlyMemory = Impl.decodeUtf8 x - static member ExpandTo(ms: System.IO.Stream, x: EncodedBody) = + static member ToStream(ms: System.IO.Stream, x: EncodedBody) = Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x +[] +type Encoder private () = + /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] - static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) + static member Uncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.FromJsonElement Encoding.DecodeToJsonElement native + FsCodec.Core.EventCodec.mapBodies Encoding.OfJsonElement Encoding.ToJsonElement native /// The body will be saved as-is under the following circumstances:
/// - the shouldCompress predicate is not satisfied for the event in question.
@@ -98,16 +112,16 @@ type Encoding private () = /// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. /// NOTE this is intended for interoperability only; a Codec (such as CodecJsonElement) that encodes to JsonElement is strongly recommended unless you don't have a choice. [] - static member EncodeUtf8TryCompress<'Event, 'Context>( + static member CompressedUtf8<'Event, 'Context>( native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?shouldCompress: Func>, bool>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function - | None -> fun _x (d: ReadOnlyMemory) -> Encoding.FromUtf8TryCompress(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.FromUtf8TryCompress(opts, d) else Encoding.FromUtf8 d - FsCodec.Core.EventCodec.mapBodies_ encode Encoding.DecodeToUtf8 native + | None -> fun _x (d: ReadOnlyMemory) -> Encoding.OfUtf8Compress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.OfUtf8Compress(opts, d) else Encoding.OfUtf8 d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToUtf8 native /// Adapts an IEventCodec rendering to JsonElement Event Bodies to attempt to compress the data.
/// The body will be saved as-is under the following circumstances:
@@ -115,25 +129,25 @@ type Encoding private () = /// - sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int produced when Encodeing conveys the encoding used, and must be round tripped alongside the body as a required input of a future Decode. [] - static member EncodeTryCompress<'Event, 'Context>( + static member Compressed<'Event, 'Context>( native: IEventCodec<'Event, JsonElement, 'Context>, [] ?shouldCompress: Func, bool>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function - | None -> fun _x (d: JsonElement) -> Encoding.FromJsonElementTryCompress(opts, d) - | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.FromJsonElementTryCompress(opts, d) else Encoding.FromJsonElement d - FsCodec.Core.EventCodec.mapBodies_ encode Encoding.DecodeToJsonElement native + | None -> fun _x (d: JsonElement) -> Encoding.OfJsonElementCompress(opts, d) + | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.OfJsonElementCompress(opts, d) else Encoding.OfJsonElement d + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToJsonElement native - /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed ReadOnlyMemory<byte>. + /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume uncompressed ReadOnlyMemory<byte>. [] - static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsUtf8<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.DecodeToUtf8 Encoding.FromUtf8 native + FsCodec.Core.EventCodec.mapBodies Encoding.ToUtf8 Encoding.OfUtf8 native - /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume Uncompressed byte[]. + /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume uncompressed byte[]. [] - static member ToUtf8ArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsUtf8ByteArray<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.mapBodies (Encoding.DecodeToUtf8 >> _.ToArray()) Encoding.FromUtf8 native + FsCodec.Core.EventCodec.mapBodies (Encoding.ToUtf8 >> _.ToArray()) Encoding.OfUtf8 native diff --git a/src/FsCodec/Encoding.fs b/src/FsCodec/Encoding.fs index 6f122f3..df6733c 100644 --- a/src/FsCodec/Encoding.fs +++ b/src/FsCodec/Encoding.fs @@ -66,39 +66,44 @@ type [] CompressionOptions = { minSize: int; minGain: int } with // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default static member Default = { minSize = 48; minGain = 4 } -[] +[] type Encoding private () = - static member FromBlob(x: ReadOnlyMemory): EncodedBody = + static member OfBlob(x: ReadOnlyMemory): EncodedBody = Impl.encodeUncompressed x - static member FromBlobTryCompress(options, x: ReadOnlyMemory): EncodedBody = + static member OfBlobCompress(options, x: ReadOnlyMemory): EncodedBody = Impl.tryCompress options.minSize options.minGain x - static member DecodeToBlob(x: EncodedBody): ReadOnlyMemory = + static member ToBlob(x: EncodedBody): ReadOnlyMemory = Impl.decode x + static member ByteCount((_encoding, data): EncodedBody) = + data.Length + +[] +type Encoder private () = /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to attempt to compress the data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. [] - static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) + static member Compressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) : IEventCodec<'Event, EncodedBody, 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.FromBlobTryCompress(opts, d)) Encoding.DecodeToBlob native + FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.OfBlobCompress(opts, d)) Encoding.ToBlob native - /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per EncodeTryCompress, but without attempting compression. + /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per Compressed, but without attempting compression. [] - static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) + static member Uncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, EncodedBody, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.FromBlob Encoding.DecodeToBlob native + FsCodec.Core.EventCodec.mapBodies Encoding.OfBlob Encoding.ToBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. [] - static member ToBlobCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsBlob<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.mapBodies Encoding.DecodeToBlob Encoding.FromBlob native + FsCodec.Core.EventCodec.mapBodies Encoding.ToBlob Encoding.OfBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. [] - static member ToBlobArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsByteArray<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.mapBodies (Encoding.DecodeToBlob >> _.ToArray()) Encoding.FromBlob native + FsCodec.Core.EventCodec.mapBodies (Encoding.ToBlob >> _.ToArray()) Encoding.OfBlob native diff --git a/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs index 9df7775..0e3545a 100644 --- a/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs +++ b/tests/FsCodec.SystemTextJson.Tests/EncodingTests.fs @@ -38,8 +38,8 @@ module InternalDecoding = let explicitBrotli = struct (2, JsonSerializer.SerializeToElement "CwuAeyJ2YWx1ZSI6IkhlbGxvIFdvcmxkIn0D") let decode useRom = - if useRom then FsCodec.SystemTextJson.Encoding.DecodeToUtf8 >> _.ToArray() >> JsonSerializer.Deserialize - else FsCodec.SystemTextJson.Encoding.DecodeToJsonElement >> JsonSerializer.Deserialize + if useRom then FsCodec.SystemTextJson.Encoding.ToUtf8 >> _.ToArray() >> JsonSerializer.Deserialize + else FsCodec.SystemTextJson.Encoding.ToJsonElement >> JsonSerializer.Deserialize let [] ``Can decode all known representations`` useRom = test <@ decode useRom direct = inputValue @> @@ -61,7 +61,7 @@ type JsonElement with member x.Utf8ByteCount = if x.ValueKind = JsonValueKind.Nu module TryCompress = - let sut = FsCodec.SystemTextJson.Encoding.EncodeTryCompress StringUtf8.sut + let sut = FsCodec.SystemTextJson.Encoder.Compressed StringUtf8.sut let compressibleValue = {| value = String('x', 5000) |} @@ -83,12 +83,13 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.SystemTextJson.Encoding.EncodeUncompressed StringUtf8.sut + let sut = FsCodec.SystemTextJson.Encoder.Uncompressed StringUtf8.sut // Borrow the value we just demonstrated to be compressible let compressibleValue = TryCompress.compressibleValue let [] roundtrips () = + let rom = ReadOnlyMemory(null : byte[]) let res' = roundtrip sut compressibleValue res' =! ValueSome compressibleValue diff --git a/tests/FsCodec.Tests/EncodingTests.fs b/tests/FsCodec.Tests/EncodingTests.fs index a73f1d8..2dcc7ce 100644 --- a/tests/FsCodec.Tests/EncodingTests.fs +++ b/tests/FsCodec.Tests/EncodingTests.fs @@ -30,7 +30,7 @@ module StringUtf8 = module TryCompress = - let sut = FsCodec.Encoding.EncodeTryCompress(StringUtf8.sut) + let sut = FsCodec.Encoder.Compressed(StringUtf8.sut) let compressibleValue = String('x', 5000) @@ -52,7 +52,7 @@ module TryCompress = module Uncompressed = - let sut = FsCodec.Encoding.EncodeUncompressed(StringUtf8.sut) + let sut = FsCodec.Encoder.Uncompressed(StringUtf8.sut) // Borrow a demonstrably compressible value let value = TryCompress.compressibleValue @@ -74,7 +74,7 @@ module Decoding = let brotli = struct(2, Convert.FromBase64String("CwWASGVsbG8gV29ybGQ=") |> ReadOnlyMemory) let [] ``Can decode all known bodies`` () = - let decode = FsCodec.Encoding.DecodeToBlob >> _.ToArray() >> Text.Encoding.UTF8.GetString + let decode = FsCodec.Encoding.ToBlob >> _.ToArray() >> Text.Encoding.UTF8.GetString test <@ decode raw = "Hello World" @> test <@ decode deflated = "Hello World" @> test <@ decode brotli = "Hello World" @> @@ -82,5 +82,5 @@ module Decoding = let [] ``Defaults to leaving the memory alone if unknown`` () = let struct(_, mem) = raw let body = struct (99, mem) - let decoded = body |> FsCodec.Encoding.DecodeToBlob |> _.ToArray() |> Text.Encoding.UTF8.GetString + let decoded = body |> FsCodec.Encoding.ToBlob |> _.ToArray() |> Text.Encoding.UTF8.GetString test <@ decoded = "Hello World" @> From 34216f1907b5a89a2010792dd05c544165e1daca Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 Jan 2025 17:38:22 +0000 Subject: [PATCH 11/11] Add ToEncodedUtf8, rename to Encodded --- src/FsCodec.Box/ByteArray.fs | 5 ++ src/FsCodec.Box/Compression.fs | 20 ++++---- src/FsCodec.SystemTextJson/Encoding.fs | 65 ++++++++++++++------------ src/FsCodec/Encoding.fs | 24 +++++----- 4 files changed, 61 insertions(+), 53 deletions(-) diff --git a/src/FsCodec.Box/ByteArray.fs b/src/FsCodec.Box/ByteArray.fs index 77316d2..914486d 100644 --- a/src/FsCodec.Box/ByteArray.fs +++ b/src/FsCodec.Box/ByteArray.fs @@ -17,6 +17,11 @@ type ByteArray private () = /// Adapt an IEventCodec that handles ReadOnlyMemory<byte> Event Bodies to instead use byte[]
/// Ideally not used as it makes pooling problematic; only provided for interop/porting scaffolding wrt Equinox V3 and EventStore.Client etc [] + static member AsByteArray<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) + : IEventCodec<'Event, byte[], 'Context> = + FsCodec.Core.EventCodec.mapBodies ByteArray.ReadOnlyMemoryToBytes ByteArray.BytesToReadOnlyMemory native + + [] static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, byte[], 'Context> = FsCodec.Core.EventCodec.mapBodies ByteArray.ReadOnlyMemoryToBytes ByteArray.BytesToReadOnlyMemory native diff --git a/src/FsCodec.Box/Compression.fs b/src/FsCodec.Box/Compression.fs index 0fc9d88..c622c0c 100644 --- a/src/FsCodec.Box/Compression.fs +++ b/src/FsCodec.Box/Compression.fs @@ -4,10 +4,6 @@ open System open System.Runtime.CompilerServices open System.Runtime.InteropServices -/// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value signifying the encoding scheme. -/// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used -type EncodedBody = (struct(int * ReadOnlyMemory)) - type [] CompressionOptions = { minSize: int; minGain: int } with static member Default = { minSize = 48; minGain = 4 } static member Uncompressed = { minSize = Int32.MaxValue; minGain = 0 } @@ -15,13 +11,13 @@ type [] CompressionOptions = { minSize: int; minGain: int } with [] type Compression private () = - static member Utf8ToEncodedDirect(x: ReadOnlyMemory): EncodedBody = + static member Utf8ToEncodedDirect(x: ReadOnlyMemory): Encoded = FsCodec.Encoding.OfBlob x - static member Utf8ToEncodedTryCompress(options, x: ReadOnlyMemory): EncodedBody = + static member Utf8ToEncodedTryCompress(options, x: ReadOnlyMemory): Encoded = FsCodec.Encoding.OfBlobCompress({ minSize = options.minSize; minGain = options.minGain }, x) - static member EncodedToUtf8(x: EncodedBody): ReadOnlyMemory = + static member EncodedToUtf8(x: Encoded): ReadOnlyMemory = FsCodec.Encoding.ToBlob x - static member EncodedToByteArray(x: EncodedBody): byte[] = + static member EncodedToByteArray(x: Encoded): byte[] = FsCodec.Encoding.ToBlob(x).ToArray() /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to attempt to compress the data.
@@ -29,7 +25,7 @@ type Compression private () = /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. [] static member EncodeTryCompress<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = let opts = defaultArg options CompressionOptions.Default let opts: FsCodec.CompressionOptions = { minSize = opts.minSize; minGain = opts.minGain } FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.OfBlobCompress(opts, d)) Encoding.ToBlob native @@ -37,17 +33,17 @@ type Compression private () = /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member EncodeUncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = Encoder.Uncompressed native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. [] - static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member ToUtf8Codec<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = Encoder.AsBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. [] - static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member ToByteArrayCodec<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, byte[], 'Context> = Encoder.AsByteArray native diff --git a/src/FsCodec.SystemTextJson/Encoding.fs b/src/FsCodec.SystemTextJson/Encoding.fs index 5c63865..843c7bd 100644 --- a/src/FsCodec.SystemTextJson/Encoding.fs +++ b/src/FsCodec.SystemTextJson/Encoding.fs @@ -9,7 +9,7 @@ open System.Text.Json /// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value identifying the encoding scheme. /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used. -type EncodedBody = (struct(int * JsonElement)) +type Encoded = (struct(int * JsonElement)) module private Impl = @@ -34,20 +34,25 @@ module private Impl = use output = new System.IO.MemoryStream() compressedBytes |> alg output output.ToArray() |> post - let decode_ direct expand struct (encoding, data: JsonElement) = + let decode_ direct expand (struct (encoding, data: JsonElement) as x) = match encoding, data.ValueKind with | Encoding.Deflate, JsonValueKind.String -> data.GetBytesFromBase64() |> expand inflateTo - | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo - | _ -> data |> direct - let decode = decode_ id (unpack InteropHelpers.Utf8ToJsonElement) + | Encoding.Brotli, JsonValueKind.String -> data.GetBytesFromBase64() |> expand brotliDecompressTo + | _ -> direct data + let decode = decode_ unbox (unpack InteropHelpers.Utf8ToJsonElement) let private blobToBase64StringJsonElement = Convert.ToBase64String >> JsonSerializer.SerializeToElement - let direct (raw: JsonElement): EncodedBody = Encoding.Direct, raw - let recode struct (encoding, data: ReadOnlyMemory): EncodedBody = + let direct (raw: JsonElement): Encoded = Encoding.Direct, raw + let ofUtf8Encoded struct (encoding, data: ReadOnlyMemory): Encoded = match encoding with | Encoding.Deflate -> Encoding.Deflate, data.ToArray() |> blobToBase64StringJsonElement - | Encoding.Brotli -> Encoding.Brotli, data.ToArray() |> blobToBase64StringJsonElement - | _ -> Encoding.Direct, data.ToArray() |> blobToBase64StringJsonElement + | Encoding.Brotli -> Encoding.Brotli, data.ToArray() |> blobToBase64StringJsonElement + | _ -> Encoding.Direct, data |> InteropHelpers.Utf8ToJsonElement let decodeUtf8 = decode_ InteropHelpers.JsonElementToUtf8 (unpack ReadOnlyMemory) + let toUtf8Encoded struct (encoding, data: JsonElement): FsCodec.Encoded = + match encoding, data.ValueKind with + | Encoding.Deflate, JsonValueKind.String -> Encoding.Deflate, data.GetBytesFromBase64() |> ReadOnlyMemory + | Encoding.Brotli, JsonValueKind.String -> Encoding.Brotli, data.GetBytesFromBase64() |> ReadOnlyMemory + | _ -> Encoding.Direct, data |> InteropHelpers.JsonElementToUtf8 (* Conditional compression logic: triggered as storage layer pulls Data/Meta fields Bodies under specified minimum size, or not meeting a required compression gain are stored directly, equivalent to if compression had not been wired in *) @@ -58,15 +63,15 @@ module private Impl = compressor.Write eventBody.Span compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing output - let compress minSize minGain (raw: JsonElement): EncodedBody = + let compress minSize minGain (raw: JsonElement): Encoded = let utf8: ReadOnlyMemory = InteropHelpers.JsonElementToUtf8 raw if utf8.Length < minSize then direct raw else let brotli = brotliCompress utf8 if utf8.Length <= int brotli.Length + minGain then direct raw else Encoding.Brotli, brotli.ToArray() |> blobToBase64StringJsonElement - let directUtf8 (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw - let compressUtf8 minSize minGain (utf8: ReadOnlyMemory): EncodedBody = + let directUtf8 (raw: ReadOnlyMemory): Encoded = Encoding.Direct, InteropHelpers.Utf8ToJsonElement raw + let compressUtf8 minSize minGain (utf8: ReadOnlyMemory): Encoded = if utf8.Length < minSize then directUtf8 utf8 else let brotli = brotliCompress utf8 @@ -76,25 +81,27 @@ module private Impl = [] type Encoding private () = - static member OfJsonElement(x: JsonElement): EncodedBody = + static member OfJsonElement(x: JsonElement): Encoded = Impl.direct x - static member OfJsonElementCompress(options, x: JsonElement): EncodedBody = + static member OfJsonElementCompress(options, x: JsonElement): Encoded = Impl.compress options.minSize options.minGain x - static member OfUtf8(x: ReadOnlyMemory): EncodedBody = + static member OfUtf8(x: ReadOnlyMemory): Encoded = Impl.directUtf8 x - static member OfUtf8Compress(options, x: ReadOnlyMemory): EncodedBody = + static member OfUtf8Compress(options, x: ReadOnlyMemory): Encoded = Impl.compressUtf8 options.minSize options.minGain x - static member OfEncodedUtf8(x: FsCodec.EncodedBody): EncodedBody = - Impl.recode x - static member ByteCount((_encoding, data): EncodedBody) = + static member OfUtf8Encoded(x: FsCodec.Encoded): Encoded = + Impl.ofUtf8Encoded x + static member ByteCount((_encoding, data): Encoded) = data.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount - static member ByteCountExpanded(x: EncodedBody) = + static member ByteCountExpanded(x: Encoded) = Impl.decode x |> _.GetRawText() |> System.Text.Encoding.UTF8.GetByteCount - static member ToJsonElement(x: EncodedBody): JsonElement = + static member ToJsonElement(x: Encoded): JsonElement = Impl.decode x - static member ToUtf8(x: EncodedBody): ReadOnlyMemory = + static member ToUtf8(x: Encoded): ReadOnlyMemory = Impl.decodeUtf8 x - static member ToStream(ms: System.IO.Stream, x: EncodedBody) = + static member ToEncodedUtf8(x: Encoded): FsCodec.Encoded = + Impl.toUtf8Encoded x + static member ToStream(ms: System.IO.Stream, x: Encoded) = Impl.decode_ (fun el -> JsonSerializer.Serialize(ms, el)) (fun dec -> dec ms) x [] @@ -103,7 +110,7 @@ type Encoder private () = /// Adapts an IEventCodec rendering to JsonElement Event Bodies to encode as per EncodeTryCompress, but without attempting compression. [] static member Uncompressed<'Event, 'Context>(native: IEventCodec<'Event, JsonElement, 'Context>) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = FsCodec.Core.EventCodec.mapBodies Encoding.OfJsonElement Encoding.ToJsonElement native /// The body will be saved as-is under the following circumstances:
@@ -116,7 +123,7 @@ type Encoder private () = native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?shouldCompress: Func>, bool>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function | None -> fun _x (d: ReadOnlyMemory) -> Encoding.OfUtf8Compress(opts, d) @@ -133,21 +140,21 @@ type Encoder private () = native: IEventCodec<'Event, JsonElement, 'Context>, [] ?shouldCompress: Func, bool>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = let opts = defaultArg options CompressionOptions.Default let encode = shouldCompress |> function | None -> fun _x (d: JsonElement) -> Encoding.OfJsonElementCompress(opts, d) | Some predicate -> fun x d -> if predicate.Invoke x then Encoding.OfJsonElementCompress(opts, d) else Encoding.OfJsonElement d - FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToJsonElement native + FsCodec.Core.EventCodec.mapBodies_ encode Encoding.ToJsonElement native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume uncompressed ReadOnlyMemory<byte>. [] - static member AsUtf8<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsUtf8<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = FsCodec.Core.EventCodec.mapBodies Encoding.ToUtf8 Encoding.OfUtf8 native /// Adapts an IEventCodec rendering to int * JsonElement Event Bodies to render and/or consume uncompressed byte[]. [] - static member AsUtf8ByteArray<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsUtf8ByteArray<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, byte[], 'Context> = FsCodec.Core.EventCodec.mapBodies (Encoding.ToUtf8 >> _.ToArray()) Encoding.OfUtf8 native diff --git a/src/FsCodec/Encoding.fs b/src/FsCodec/Encoding.fs index df6733c..4cf3cc1 100644 --- a/src/FsCodec/Encoding.fs +++ b/src/FsCodec/Encoding.fs @@ -6,7 +6,7 @@ open System.Runtime.InteropServices /// Represents the body of an Event (or its Metadata), holding the encoded form of the buffer together with an enum value signifying the encoding scheme. /// Enables the decoding side to transparently inflate the data on loading without burdening the application layer with tracking the encoding scheme used -type EncodedBody = (struct(int * ReadOnlyMemory)) +type Encoded = (struct(int * ReadOnlyMemory)) module Encoding = let [] Direct = 0 // Assumed for all values not listed here @@ -28,7 +28,7 @@ module private Impl = let input = new System.IO.MemoryStream(data.ToArray(), writable = false) let decompressor = new System.IO.Compression.DeflateStream(input, System.IO.Compression.CompressionMode.Decompress, leaveOpen = true) decompressor.CopyTo output - let private brotliDecompressTo output (data: ReadOnlyMemory) = + let private brotliDecompressTo output (data: ReadOnlyMemory) = let input = new System.IO.MemoryStream(data.ToArray(), writable = false) use decompressor = new System.IO.Compression.BrotliStream(input, System.IO.Compression.CompressionMode.Decompress) decompressor.CopyTo output @@ -51,8 +51,8 @@ module private Impl = compressor.Write(eventBody.Span) compressor.Close() // NOTE Close, not Flush; we want the output fully terminated to reduce surprises when decompressing output - let encodeUncompressed (raw: ReadOnlyMemory): EncodedBody = Encoding.Direct, raw - let tryCompress minSize minGain (raw: ReadOnlyMemory): EncodedBody = + let encodeUncompressed (raw: ReadOnlyMemory): Encoded = Encoding.Direct, raw + let tryCompress minSize minGain (raw: ReadOnlyMemory): Encoded = if raw.Length < minSize then encodeUncompressed raw else match brotliCompress raw with | tmp when raw.Length > int tmp.Length + minGain -> Encoding.Brotli, tmp.ToArray() |> ReadOnlyMemory @@ -69,13 +69,13 @@ type [] CompressionOptions = { minSize: int; minGain: int } with [] type Encoding private () = - static member OfBlob(x: ReadOnlyMemory): EncodedBody = + static member OfBlob(x: ReadOnlyMemory): Encoded = Impl.encodeUncompressed x - static member OfBlobCompress(options, x: ReadOnlyMemory): EncodedBody = + static member OfBlobCompress(options, x: ReadOnlyMemory): Encoded = Impl.tryCompress options.minSize options.minGain x - static member ToBlob(x: EncodedBody): ReadOnlyMemory = + static member ToBlob(x: Encoded): ReadOnlyMemory = Impl.decode x - static member ByteCount((_encoding, data): EncodedBody) = + static member ByteCount((_encoding, data): Encoded) = data.Length [] @@ -86,24 +86,24 @@ type Encoder private () = /// The int conveys a value that must be round tripped alongside the body in order for the decoding process to correctly interpret it. [] static member Compressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = let opts = defaultArg options CompressionOptions.Default FsCodec.Core.EventCodec.mapBodies (fun d -> Encoding.OfBlobCompress(opts, d)) Encoding.ToBlob native /// Adapts an IEventCodec rendering to ReadOnlyMemory<byte> Event Bodies to encode as per Compressed, but without attempting compression. [] static member Uncompressed<'Event, 'Context>(native: IEventCodec<'Event, ReadOnlyMemory, 'Context>) - : IEventCodec<'Event, EncodedBody, 'Context> = + : IEventCodec<'Event, Encoded, 'Context> = FsCodec.Core.EventCodec.mapBodies Encoding.OfBlob Encoding.ToBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory<byte>. [] - static member AsBlob<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsBlob<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = FsCodec.Core.EventCodec.mapBodies Encoding.ToBlob Encoding.OfBlob native /// Adapts an IEventCodec rendering to int * ReadOnlyMemory<byte> Event Bodies to render and/or consume from Uncompressed byte[]. [] - static member AsByteArray<'Event, 'Context>(native: IEventCodec<'Event, EncodedBody, 'Context>) + static member AsByteArray<'Event, 'Context>(native: IEventCodec<'Event, Encoded, 'Context>) : IEventCodec<'Event, byte[], 'Context> = FsCodec.Core.EventCodec.mapBodies (Encoding.ToBlob >> _.ToArray()) Encoding.OfBlob native