Add Client Reports feature

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Collect and send Client Reports to Sentry, which contain counts of discarded events. ([#1556](https://github.com/getsentry/sentry-dotnet/pull/1556))
 - Use a default value of 60 seconds if a `Retry-After` header is not present. ([#1537](https://github.com/getsentry/sentry-dotnet/pull/1537))
 
 ### Fixes

src/Sentry/Envelopes/Envelope.cs

@@ -237,5 +237,12 @@ public static async Task<Envelope> DeserializeAsync(
 
             return new Envelope(header, items);
         }
+
+        internal Envelope WithClientReport(ClientReport clientReport)
+        {
+            var items = Items.ToList();
+            items.Add(EnvelopeItem.FromClientReport(clientReport));
+            return new Envelope(Header, items);
+        }
     }
 }

src/Sentry/Envelopes/EnvelopeItem.cs

@@ -21,6 +21,7 @@ public sealed class EnvelopeItem : ISerializable, IDisposable
         private const string TypeValueTransaction = "transaction";
         private const string TypeValueSession = "session";
         private const string TypeValueAttachment = "attachment";
+        private const string TypeValueClientReport = "client_report";
         private const string LengthKey = "length";
         private const string FileNameKey = "filename";
 
@@ -34,6 +35,15 @@ public sealed class EnvelopeItem : ISerializable, IDisposable
         /// </summary>
         public ISerializable Payload { get; }
 
+        internal DataCategory DataCategory => TryGetType() switch
+        {
+            TypeValueEvent => DataCategory.Error,
+            TypeValueTransaction => DataCategory.Transaction,
+            TypeValueSession => DataCategory.Session,
+            TypeValueAttachment => DataCategory.Attachment,
+            _ => DataCategory.Default
+        };
+
         /// <summary>
         /// Initializes an instance of <see cref="EnvelopeItem"/>.
         /// </summary>
@@ -215,6 +225,19 @@ public static EnvelopeItem FromAttachment(Attachment attachment)
             return new EnvelopeItem(header, new StreamSerializable(stream));
         }
 
+        /// <summary>
+        /// Creates an envelope item from a client report.
+        /// </summary>
+        internal static EnvelopeItem FromClientReport(ClientReport clientReport)
+        {
+            var header = new Dictionary<string, object?>(1, StringComparer.Ordinal)
+            {
+                [TypeKey] = TypeValueClientReport
+            };
+
+            return new EnvelopeItem(header, new JsonSerializable(clientReport));
+        }
+
         private static async Task<IReadOnlyDictionary<string, object?>> DeserializeHeaderAsync(
             Stream stream,
             CancellationToken cancellationToken = default)

src/Sentry/Internal/BackgroundWorker.cs

@@ -1,9 +1,12 @@
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Sentry.Extensibility;
+using Sentry.Infrastructure;
 using Sentry.Protocol.Envelopes;
 
 namespace Sentry.Internal
@@ -13,9 +16,11 @@ internal class BackgroundWorker : IBackgroundWorker, IDisposable
         private readonly ITransport _transport;
         private readonly SentryOptions _options;
         private readonly ConcurrentQueue<Envelope> _queue;
+        private readonly ISystemClock _clock;
         private readonly int _maxItems;
         private readonly CancellationTokenSource _shutdownSource;
         private readonly SemaphoreSlim _queuedEnvelopeSemaphore;
+        private readonly ThreadsafeCounterDictionary<(DataCategory Category, DiscardReason Reason)> _discardedEvents = new();
 
         private volatile bool _disposed;
         private int _currentItems;
@@ -24,6 +29,8 @@ internal class BackgroundWorker : IBackgroundWorker, IDisposable
 
         internal Task WorkerTask { get; }
 
+        internal IReadOnlyDictionary<(DataCategory Category, DiscardReason Reason), int> DiscardedEvents => _discardedEvents;
+
         public int QueuedItems => _queue.Count;
 
         public BackgroundWorker(
@@ -37,11 +44,13 @@ internal BackgroundWorker(
             ITransport transport,
             SentryOptions options,
             CancellationTokenSource? shutdownSource = null,
-            ConcurrentQueue<Envelope>? queue = null)
+            ConcurrentQueue<Envelope>? queue = null,
+            ISystemClock? clock = null)
         {
             _transport = transport;
             _options = options;
             _queue = queue ?? new ConcurrentQueue<Envelope>();
+            _clock = clock ?? new SystemClock();
             _maxItems = options.MaxQueueItems;
             _shutdownSource = shutdownSource ?? new CancellationTokenSource();
             _queuedEnvelopeSemaphore = new SemaphoreSlim(0, _maxItems);
@@ -59,6 +68,10 @@ public bool EnqueueEnvelope(Envelope envelope)
             if (Interlocked.Increment(ref _currentItems) > _maxItems)
             {
                 _ = Interlocked.Decrement(ref _currentItems);
+                foreach (var item in envelope.Items)
+                {
+                    _discardedEvents.Increment((item.DataCategory, DiscardReason.QueueOverflow));
+                }
                 return false;
             }
 
@@ -121,6 +134,22 @@ private async Task WorkerAsync()
                             // Dispose inside try/catch
                             using var _ = envelope;
 
+                            // Read and reset discards even if we're not sending them (to prevent excessive growth over time)
+                            var discardedEvents = _discardedEvents.ReadAllAndReset();
+
+                            // Create and attach the client report
+                            if (_options.SendClientReports && discardedEvents.Any(x => x.Value > 0))
+                            {
+                                var timestamp = _clock.GetUtcNow();
+                                var clientReport = new ClientReport(timestamp, discardedEvents);
+                                envelope = envelope.WithClientReport(clientReport);
+
+                                _options.LogDebug(
+                                    "Attached client report to envelope {0}.",
+                                    envelope.TryGetEventId());
+                            }
+
+                            // Send the envelope
                             var task = _transport.SendEnvelopeAsync(envelope, shutdownTimeout.Token);
 
                             _options.LogDebug(

src/Sentry/Internal/ClientReport.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using Sentry.Extensibility;
+using Sentry.Internal.Extensions;
+
+namespace Sentry.Internal
+{
+    internal class ClientReport : IJsonSerializable
+    {
+        public DateTimeOffset Timestamp { get; }
+        public IReadOnlyDictionary<(DataCategory Category, DiscardReason Reason), int> DiscardedEvents { get; }
+
+        public ClientReport(DateTimeOffset timestamp,
+            IReadOnlyDictionary<(DataCategory Category, DiscardReason Reason), int> discardedEvents)
+        {
+            Timestamp = timestamp;
+            DiscardedEvents = discardedEvents;
+        }
+
+        public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
+        {
+            writer.WriteStartObject();
+
+            writer.WriteString("timestamp", Timestamp);
+
+            writer.WriteStartArray("discarded_events");
+            foreach (var ((category, reason), value) in DiscardedEvents.Where(x=> x.Value > 0))
+            {
+                writer.WriteStartObject();
+                writer.WriteString("reason", reason);
+                writer.WriteString("category", category);
+                writer.WriteNumber("quantity", value);
+                writer.WriteEndObject();
+            }
+            writer.WriteEndArray();
+
+            writer.WriteEndObject();
+        }
+    }
+}

src/Sentry/Internal/DataCategory.cs

@@ -0,0 +1,17 @@
+namespace Sentry.Internal
+{
+    internal record DataCategory : Enumeration
+    {
+        public static DataCategory Default = new("default");
+        public static DataCategory Error = new("error");
+        public static DataCategory Transaction = new("transaction");
+        public static DataCategory Security = new("security");
+        public static DataCategory Attachment = new("attachment");
+        public static DataCategory Session = new("session");
+        public static DataCategory Internal = new("internal");
+
+        private DataCategory(string value) : base(value)
+        {
+        }
+    }
+}

src/Sentry/Internal/DiscardReason.cs

@@ -0,0 +1,15 @@
+namespace Sentry.Internal
+{
+    internal record DiscardReason : Enumeration
+    {
+        public static DiscardReason QueueOverflow = new("queue_overflow");
+        public static DiscardReason CacheOverflow = new("cache_overflow");
+        public static DiscardReason RateLimitBackoff = new("ratelimit_backoff");
+        public static DiscardReason NetworkError = new("network_error");
+        public static DiscardReason SampleRate = new("sample_rate");
+
+        private DiscardReason(string value) : base(value)
+        {
+        }
+    }
+}

src/Sentry/Internal/Enumeration.cs

@@ -0,0 +1,9 @@
+namespace Sentry.Internal
+{
+    internal abstract record Enumeration(string Value)
+    {
+        public string Value { get; } = Value;
+
+        public override string ToString() => Value;
+    }
+}

src/Sentry/Internal/Extensions/JsonExtensions.cs

@@ -523,5 +523,20 @@ public static void WriteDynamicIfNotNull(
                 writer.WriteDynamic(propertyName, value, logger);
             }
         }
+
+        public static void WriteString(
+            this Utf8JsonWriter writer,
+            string propertyName,
+            Enumeration? value)
+        {
+            if (value == null)
+            {
+                writer.WriteNull(propertyName);
+            }
+            else
+            {
+                writer.WriteString(propertyName, value.Value);
+            }
+        }
     }
 }

src/Sentry/Internal/ThreadsafeCounterDictionary.cs

@@ -0,0 +1,110 @@
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Threading;
+
+namespace Sentry.Internal
+{
+    /// <summary>
+    /// Provides a keyed set of counters that can be incremented, read, and reset atomically.
+    /// </summary>
+    /// <typeparam name="TKey">The type of the key.</typeparam>
+    internal class ThreadsafeCounterDictionary<TKey> : IReadOnlyDictionary<TKey, int>
+        where TKey : notnull
+    {
+        private class CounterItem
+        {
+            private int _value;
+            public int Value => _value;
+
+            public void Increment() => Interlocked.Increment(ref _value);
+
+            public int ReadAndReset() => Interlocked.Exchange(ref _value, 0);
+        }
+
+        private readonly ConcurrentDictionary<TKey, CounterItem> _items = new();
+
+        /// <summary>
+        /// Atomically increments a counter based on the key provided, creating the counter if necessary.
+        /// </summary>
+        /// <param name="key">The key of the counter to increment.</param>
+        public void Increment(TKey key) => _items.GetOrAdd(key, new CounterItem()).Increment();
+
+        /// <summary>
+        /// Gets a single counter's value while atomically resetting it to zero.
+        /// </summary>
+        /// <param name="key">The key to the counter.</param>
+        /// <returns>The previous value of the counter.</returns>
+        /// <remarks>If no counter with the given key has been set, this returns zero.</remarks>
+        public int ReadAndReset(TKey key) => _items.TryGetValue(key, out var item) ? item.ReadAndReset() : 0;
+
+        /// <summary>
+        /// Gets the keys and values of all of the counters while atomically resetting them to zero.
+        /// </summary>
+        /// <returns>A read-only dictionary containing the key and the previous value for each counter.</returns>
+        public IReadOnlyDictionary<TKey, int> ReadAllAndReset()
+        {
+            // Read all the counters while atomically resetting them to zero
+            var counts = _items.ToDictionary(
+                x => x.Key,
+                x => x.Value.ReadAndReset());
+
+            return new ReadOnlyDictionary<TKey, int>(counts);
+        }
+
+        /// <summary>
+        /// Gets an enumerator over the keys and values of the counters.
+        /// </summary>
+        /// <returns>An enumerator.</returns>
+        public IEnumerator<KeyValuePair<TKey, int>> GetEnumerator() => _items
+            .Select(x => new KeyValuePair<TKey, int>(x.Key, x.Value.Value))
+            .GetEnumerator();
+
+        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+        /// <summary>
+        /// Gets the number of counters currently being tracked.
+        /// </summary>
+        public int Count => _items.Count;
+
+        /// <summary>
+        /// Tests whether or not a counter with the given key exists.
+        /// </summary>
+        /// <param name="key">The key to check.</param>
+        /// <returns>True if the counter exists, false otherwise.</returns>
+        public bool ContainsKey(TKey key) => _items.ContainsKey(key);
+
+        /// <summary>
+        /// Gets the current value of the counter specified.
+        /// </summary>
+        /// <param name="key">The key of the counter.</param>
+        /// <param name="value">The value of the counter, or zero if the counter does not yet exist.</param>
+        /// <returns>Returns <c>true</c> in all cases.</returns>
+        public bool TryGetValue(TKey key, out int value)
+        {
+            value = this[key];
+            return true;
+        }
+
+        /// <summary>
+        /// Gets the current value of the counter specified, returning zero if the counter does not yet exist.
+        /// </summary>
+        /// <param name="key">The key of the counter.</param>
+        public int this[TKey key] => _items.TryGetValue(key, out var item) ? item.Value : 0;
+
+        /// <summary>
+        /// Gets all of the current counter keys.
+        /// </summary>
+        public IEnumerable<TKey> Keys => _items.Keys;
+
+        /// <summary>
+        /// Gets all of the current counter values.
+        /// </summary>
+        /// <remarks>
+        /// Useless, but required by the IReadOnlyDictionary interface.
+        /// </remarks>
+        public IEnumerable<int> Values => _items.Values.Select(x => x.Value);
+    }
+}

src/Sentry/SentryOptions.cs

@@ -333,6 +333,12 @@ public int MaxCacheItems
         /// <see href="https://github.com/getsentry/sentry-dotnet/issues/71"/>
         public bool RequestBodyCompressionBuffered { get; set; } = true;
 
+        /// <summary>
+        /// Whether to send client reports, which contain statistics about discarded events.
+        /// </summary>
+        /// <see href="https://develop.sentry.dev/sdk/client-reports/"/>
+        public bool SendClientReports { get; set; } = true;
+
         /// <summary>
         /// An optional web proxy
         /// </summary>

test/Sentry.DiagnosticSource.Tests/ApiApprovalTests.Run.Core3_1.verified.txt

@@ -476,6 +476,7 @@ namespace Sentry
         public System.IO.Compression.CompressionLevel RequestBodyCompressionLevel { get; set; }
         public float? SampleRate { get; set; }
         public Sentry.IScopeObserver? ScopeObserver { get; set; }
+        public bool SendClientReports { get; set; }
         public bool SendDefaultPii { get; set; }
         public Sentry.ISentryScopeStateProcessor SentryScopeStateProcessor { get; set; }
         public string? ServerName { get; set; }

test/Sentry.DiagnosticSource.Tests/ApiApprovalTests.Run.DotNet5_0.verified.txt

@@ -476,6 +476,7 @@ namespace Sentry
         public System.IO.Compression.CompressionLevel RequestBodyCompressionLevel { get; set; }
         public float? SampleRate { get; set; }
         public Sentry.IScopeObserver? ScopeObserver { get; set; }
+        public bool SendClientReports { get; set; }
         public bool SendDefaultPii { get; set; }
         public Sentry.ISentryScopeStateProcessor SentryScopeStateProcessor { get; set; }
         public string? ServerName { get; set; }