[CIVisibility] - Add support for GZip compression in Multipart payloads

tracer/src/Datadog.Trace/Agent/IMultipartApiRequest.cs

@@ -6,11 +6,12 @@
 #nullable enable
 
 using System.Threading.Tasks;
+using Datadog.Trace.Agent.Transports;
 
 namespace Datadog.Trace.Agent
 {
     internal interface IMultipartApiRequest
     {
-        Task<IApiResponse> PostAsync(params MultipartFormItem[] items);
+        Task<IApiResponse> PostAsync(MultipartFormItem[] items, MultipartCompression multipartCompression = MultipartCompression.None);
     }
 }

tracer/src/Datadog.Trace/Agent/Transports/ApiWebRequest.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.IO;
+using System.IO.Compression;
 using System.Net;
 using System.Text;
 using System.Threading.Tasks;
@@ -74,8 +75,9 @@ public async Task<IApiResponse> PostAsync(Func<Stream, Task> writeToRequestStrea
         /// WARNING: Name and FileName of each MultipartFormItem instance must be ASCII encoding compatible.
         /// </summary>
         /// <param name="items">Multipart form data items</param>
+        /// <param name="multipartCompression">Multipart compression</param>
         /// <returns>Task with the response</returns>
-        public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
+        public async Task<IApiResponse> PostAsync(MultipartFormItem[] items, MultipartCompression multipartCompression = MultipartCompression.None)
         {
             if (items is null)
             {
@@ -84,17 +86,33 @@ public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
 
             Log.Debug<int>("Sending multipart form request with {Count} items.", items.Length);
 
-            ResetRequest(method: "POST", contentType: "multipart/form-data; boundary=" + Boundary, contentEncoding: null);
+            ResetRequest(method: "POST", contentType: "multipart/form-data; boundary=" + Boundary, contentEncoding: multipartCompression == MultipartCompression.GZip ? "gzip" : null);
+            using (var reqStream = await _request.GetRequestStreamAsync().ConfigureAwait(false))
+            {
+                if (multipartCompression == MultipartCompression.GZip)
+                {
+                    Log.Debug("Using MultipartCompression.GZip");
+                    using var gzip = new GZipStream(reqStream, CompressionMode.Compress, true);
+                    await WriteToStreamAsync(items, gzip).ConfigureAwait(false);
+                    Log.Debug("Compressing multipart payload...");
+                }
+                else
+                {
+                    await WriteToStreamAsync(items, reqStream).ConfigureAwait(false);
+                }
+            }
 
-            using (var requestStream = await _request.GetRequestStreamAsync().ConfigureAwait(false))
+            return await FinishAndGetResponse().ConfigureAwait(false);
+
+            async Task WriteToStreamAsync(MultipartFormItem[] multipartItems, Stream requestStream)
             {
                 // Write form request using the boundary
                 var boundaryBytes = _boundarySeparatorInBytes ??= Encoding.ASCII.GetBytes(BoundarySeparator);
                 var trailerBytes = _boundaryTrailerInBytes ??= Encoding.ASCII.GetBytes(BoundaryTrailer);
 
                 // Write each MultipartFormItem
                 var itemsWritten = 0;
-                foreach (var item in items)
+                foreach (var item in multipartItems)
                 {
                     byte[] headerBytes = null;
 
@@ -159,8 +177,6 @@ public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
                     await requestStream.WriteAsync(trailerBytes, 0, trailerBytes.Length).ConfigureAwait(false);
                 }
             }
-
-            return await FinishAndGetResponse().ConfigureAwait(false);
         }
 
         private void ResetRequest(string method, string contentType, string contentEncoding)

tracer/src/Datadog.Trace/Agent/Transports/GzipCompressedContent.cs

@@ -0,0 +1,52 @@
+// <copyright file="GzipCompressedContent.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+#if NETCOREAPP
+using System.IO;
+using System.IO.Compression;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Datadog.Trace.Logging;
+
+namespace Datadog.Trace.Agent.Transports;
+
+internal class GzipCompressedContent : HttpContent
+{
+    private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<GzipCompressedContent>();
+
+    private readonly HttpContent _content;
+
+    public GzipCompressedContent(HttpContent content)
+    {
+        // Copy original headers
+        foreach (var header in content.Headers)
+        {
+            Headers.TryAddWithoutValidation(header.Key, header.Value);
+        }
+
+        Headers.ContentEncoding.Add("gzip");
+        _content = content;
+    }
+
+    protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context)
+    {
+        Log.Debug("GZip compressing payload...");
+#pragma warning disable CA2007
+        await using var gzip = new GZipStream(stream, CompressionMode.Compress, true);
+#pragma warning restore CA2007
+        await _content.CopyToAsync(gzip).ConfigureAwait(false);
+    }
+
+    protected override bool TryComputeLength(out long length)
+    {
+        length = -1;
+        return false;
+    }
+}
+
+#endif

tracer/src/Datadog.Trace/Agent/Transports/HttpClientRequest.cs

@@ -123,7 +123,7 @@ public async Task<IApiResponse> PostAsync(Func<Stream, Task> writeToRequestStrea
             return new HttpClientResponse(response);
         }
 
-        public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
+        public async Task<IApiResponse> PostAsync(MultipartFormItem[] items, MultipartCompression multipartCompression = MultipartCompression.None)
         {
             if (items is null)
             {
@@ -133,8 +133,6 @@ public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
             Log.Debug<int>("Sending multipart form request with {Count} items.", items.Length);
 
             using var formDataContent = new MultipartFormDataContent(boundary: Boundary);
-            _postRequest.Content = formDataContent;
-
             foreach (var item in items)
             {
                 HttpContent content = null;
@@ -166,6 +164,16 @@ public async Task<IApiResponse> PostAsync(params MultipartFormItem[] items)
                 }
             }
 
+            if (multipartCompression == MultipartCompression.GZip)
+            {
+                Log.Debug("Using MultipartCompression.GZip");
+                _postRequest.Content = new GzipCompressedContent(formDataContent);
+            }
+            else
+            {
+                _postRequest.Content = formDataContent;
+            }
+
             var response = await _client.SendAsync(_postRequest).ConfigureAwait(false);
             return new HttpClientResponse(response);
         }

tracer/src/Datadog.Trace/Agent/Transports/MultipartCompression.cs

@@ -0,0 +1,14 @@
+// <copyright file="MultipartCompression.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+namespace Datadog.Trace.Agent.Transports;
+
+internal enum MultipartCompression
+{
+    None,
+    GZip
+}

tracer/src/Datadog.Trace/Ci/Agent/CIWriterHttpSender.cs

@@ -255,7 +255,7 @@ await SendPayloadAsync(
                 {
                     if (request is IMultipartApiRequest multipartRequest)
                     {
-                        return multipartRequest.PostAsync(payloadArray);
+                        return multipartRequest.PostAsync(payloadArray, MultipartCompression.GZip);
                     }
 
                     MultipartApiRequestNotSupported.Throw();

tracer/src/Datadog.Trace/Ci/IntelligentTestRunnerClient.cs

@@ -583,9 +583,9 @@ async Task<long> InternalSendObjectsPackFileAsync(string packFile, bool finalTry
 
                 try
                 {
-                    using var response = await multipartRequest.PostAsync(
+                    using var response = await multipartRequest.PostAsync([
                                                                     new MultipartFormItem("pushedSha", MimeTypes.Json, null, new ArraySegment<byte>(jsonPushedShaBytes)),
-                                                                    new MultipartFormItem("packfile", "application/octet-stream", null, fileStream))
+                                                                    new MultipartFormItem("packfile", "application/octet-stream", null, fileStream)])
                                                                .ConfigureAwait(false);
                     var responseContent = await response.ReadAsStringAsync().ConfigureAwait(false);
                     if (TelemetryHelper.GetErrorTypeFromStatusCode(response.StatusCode) is { } errorType)