[OTLP] Export LogRecord.CategoryName as InstrumentationScope name (#4941

)

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md

@@ -19,6 +19,12 @@ attributes will be exported when
 variable will be set to `true`.
 ([#4892](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4892))
 
+* `LogRecord.CategoryName` will now be exported as
+[InstrumentationScope](https://github.com/open-telemetry/opentelemetry-dotnet/blob/3c2bb7c93dd2e697636479a1882f49bb0c4a362e/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/opentelemetry/proto/common/v1/common.proto#L71-L81)
+`name` field under
+[ScopeLogs](https://github.com/open-telemetry/opentelemetry-dotnet/blob/3c2bb7c93dd2e697636479a1882f49bb0c4a362e/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/opentelemetry/proto/logs/v1/logs.proto#L64-L75).
+([#4941](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4941))
+
 ## 1.6.0
 
 Released 2023-Sep-05

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OtlpLogRecordTransformer.cs

@@ -14,6 +14,7 @@
 // limitations under the License.
 // </copyright>
 
+using System.Collections.Concurrent;
 using System.Runtime.CompilerServices;
 using Google.Protobuf;
 using OpenTelemetry.Internal;
@@ -28,6 +29,8 @@ namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 
 internal sealed class OtlpLogRecordTransformer
 {
+    internal static readonly ConcurrentBag<OtlpLogs.ScopeLogs> LogListPool = new();
+
     private readonly SdkLimitOptions sdkLimitOptions;
     private readonly ExperimentalOptions experimentalOptions;
 
@@ -41,6 +44,9 @@ internal OtlpCollector.ExportLogsServiceRequest BuildExportRequest(
         OtlpResource.Resource processResource,
         in Batch<LogRecord> logRecordBatch)
     {
+        // TODO: https://github.com/open-telemetry/opentelemetry-dotnet/issues/4943
+        Dictionary<string, OtlpLogs.ScopeLogs> logsByCategory = new Dictionary<string, OtlpLogs.ScopeLogs>();
+
         var request = new OtlpCollector.ExportLogsServiceRequest();
 
         var resourceLogs = new OtlpLogs.ResourceLogs
@@ -49,21 +55,64 @@ internal OtlpCollector.ExportLogsServiceRequest BuildExportRequest(
         };
         request.ResourceLogs.Add(resourceLogs);
 
-        var scopeLogs = new OtlpLogs.ScopeLogs();
-        resourceLogs.ScopeLogs.Add(scopeLogs);
-
         foreach (var logRecord in logRecordBatch)
         {
             var otlpLogRecord = this.ToOtlpLog(logRecord);
             if (otlpLogRecord != null)
             {
+                if (!logsByCategory.TryGetValue(logRecord.CategoryName, out var scopeLogs))
+                {
+                    scopeLogs = this.GetLogListFromPool(logRecord.CategoryName);
+                    logsByCategory.Add(logRecord.CategoryName, scopeLogs);
+                    resourceLogs.ScopeLogs.Add(scopeLogs);
+                }
+
                 scopeLogs.LogRecords.Add(otlpLogRecord);
             }
         }
 
         return request;
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal void Return(OtlpCollector.ExportLogsServiceRequest request)
+    {
+        var resourceLogs = request.ResourceLogs.FirstOrDefault();
+        if (resourceLogs == null)
+        {
+            return;
+        }
+
+        foreach (var scope in resourceLogs.ScopeLogs)
+        {
+            scope.LogRecords.Clear();
+            LogListPool.Add(scope);
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal OtlpLogs.ScopeLogs GetLogListFromPool(string name)
+    {
+        if (!LogListPool.TryTake(out var logs))
+        {
+            logs = new OtlpLogs.ScopeLogs
+            {
+                Scope = new OtlpCommon.InstrumentationScope
+                {
+                    Name = name, // Name is enforced to not be null, but it can be empty.
+                    Version = string.Empty, // proto requires this to be non-null.
+                },
+            };
+        }
+        else
+        {
+            logs.Scope.Name = name;
+            logs.Scope.Version = string.Empty;
+        }
+
+        return logs;
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     internal OtlpLogs.LogRecord ToOtlpLog(LogRecord logRecord)
     {

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs

@@ -90,9 +90,11 @@ public override ExportResult Export(in Batch<LogRecord> logRecordBatch)
         // Prevents the exporter's gRPC and HTTP operations from being instrumented.
         using var scope = SuppressInstrumentationScope.Begin();
 
+        OtlpCollector.ExportLogsServiceRequest request = null;
+
         try
         {
-            var request = this.otlpLogRecordTransformer.BuildExportRequest(this.ProcessResource, logRecordBatch);
+            request = this.otlpLogRecordTransformer.BuildExportRequest(this.ProcessResource, logRecordBatch);
 
             if (!this.exportClient.SendExportRequest(request))
             {
@@ -104,6 +106,13 @@ public override ExportResult Export(in Batch<LogRecord> logRecordBatch)
             OpenTelemetryProtocolExporterEventSource.Log.ExportMethodException(ex);
             return ExportResult.Failure;
         }
+        finally
+        {
+            if (request != null)
+            {
+                this.otlpLogRecordTransformer.Return(request);
+            }
+        }
 
         return ExportResult.Success;
     }

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpLogExporterTests.cs

@@ -26,6 +26,7 @@
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Logs;
+using OpenTelemetry.Resources;
 using OpenTelemetry.Tests;
 using OpenTelemetry.Trace;
 using Xunit;
@@ -1242,6 +1243,64 @@ public void AddOtlpLogExporterLogRecordProcessorOptionsTest(ExportProcessorType
         }
     }
 
+    [Fact]
+    public void ValidateInstrumentationScope()
+    {
+        var logRecords = new List<LogRecord>();
+        using var loggerFactory = LoggerFactory.Create(builder =>
+        {
+            builder
+                .AddOpenTelemetry(options => options
+                    .AddInMemoryExporter(logRecords));
+        });
+
+        var logger1 = loggerFactory.CreateLogger("OtlpLogExporterTests-A");
+        logger1.LogInformation("Hello from red-tomato");
+
+        var logger2 = loggerFactory.CreateLogger("OtlpLogExporterTests-B");
+        logger2.LogInformation("Hello from green-tomato");
+
+        Assert.Equal(2, logRecords.Count);
+
+        var batch = new Batch<LogRecord>(logRecords.ToArray(), logRecords.Count);
+        var logRecordTransformer = new OtlpLogRecordTransformer(new(), new());
+
+        var resourceBuilder = ResourceBuilder.CreateEmpty();
+        var processResource = resourceBuilder.Build().ToOtlpResource();
+
+        var request = logRecordTransformer.BuildExportRequest(processResource, batch);
+
+        Assert.Single(request.ResourceLogs);
+
+        var scope1 = request.ResourceLogs[0].ScopeLogs.First();
+        var scope2 = request.ResourceLogs[0].ScopeLogs.Last();
+
+        Assert.Equal("OtlpLogExporterTests-A", scope1.Scope.Name);
+        Assert.Equal("OtlpLogExporterTests-B", scope2.Scope.Name);
+
+        Assert.Single(scope1.LogRecords);
+        Assert.Single(scope2.LogRecords);
+
+        var logrecord1 = scope1.LogRecords[0];
+        var logrecord2 = scope2.LogRecords[0];
+
+        Assert.Equal("Hello from red-tomato", logrecord1.Body.StringValue);
+
+        Assert.Equal("Hello from green-tomato", logrecord2.Body.StringValue);
+
+        // Validate LogListPool
+        Assert.Empty(OtlpLogRecordTransformer.LogListPool);
+        logRecordTransformer.Return(request);
+        Assert.Equal(2, OtlpLogRecordTransformer.LogListPool.Count);
+
+        request = logRecordTransformer.BuildExportRequest(processResource, batch);
+
+        Assert.Single(request.ResourceLogs);
+
+        // ScopeLogs will be reused.
+        Assert.Empty(OtlpLogRecordTransformer.LogListPool);
+    }
+
     private static OtlpCommon.KeyValue TryGetAttribute(OtlpLogs.LogRecord record, string key)
     {
         return record.Attributes.FirstOrDefault(att => att.Key == key);