Remove authority from URLs sent to Sentry

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Remove authority from URLs sent to Sentry ([#2365](https://github.com/getsentry/sentry-dotnet/pull/2365))
 - Add `Hint` support  ([#2351](https://github.com/getsentry/sentry-dotnet/pull/2351))
   - Currently, this allows you to manipulate attachments in the various "before" event delegates.
   - Hints can also be used in event and transaction processors by implementing `ISentryEventProcessorWithHint` or `ISentryTransactionProcessorWithHint`, instead of `ISentryEventProcessor` or `ISentryTransactionProcessor`.

src/Sentry/Breadcrumb.cs

@@ -1,4 +1,5 @@
 using Sentry.Extensibility;
+using Sentry.Internal;
 using Sentry.Internal.Extensions;
 
 namespace Sentry;
@@ -9,6 +10,12 @@ namespace Sentry;
 [DebuggerDisplay("Message: {" + nameof(Message) + "}, Type: {" + nameof(Type) + "}")]
 public sealed class Breadcrumb : IJsonSerializable
 {
+    private readonly IReadOnlyDictionary<string, string>? _data;
+    private readonly string? _message;
+
+    private bool _sendDefaultPii = true;
+    internal void Redact() => _sendDefaultPii = false;
+
     /// <summary>
     /// A timestamp representing when the breadcrumb occurred.
     /// </summary>
@@ -21,7 +28,11 @@ public sealed class Breadcrumb : IJsonSerializable
     /// If a message is provided, it’s rendered as text and the whitespace is preserved.
     /// Very long text might be abbreviated in the UI.
     /// </summary>
-    public string? Message { get; }
+    public string? Message
+    {
+        get => _sendDefaultPii ? _message : _message?.RedactUrl();
+        private init => _message = value;
+    }
 
     /// <summary>
     /// The type of breadcrumb.
@@ -39,7 +50,17 @@ public sealed class Breadcrumb : IJsonSerializable
     /// Contains a sub-object whose contents depend on the breadcrumb type.
     /// Additional parameters that are unsupported by the type are rendered as a key/value table.
     /// </remarks>
-    public IReadOnlyDictionary<string, string>? Data { get; }
+    public IReadOnlyDictionary<string, string>? Data
+    {
+        get => _sendDefaultPii
+            ? _data
+            : _data?.ToDictionary(
+                x => x.Key,
+                x => x.Value.RedactUrl()
+                )
+            ;
+        private init => _data = value;
+    }
 
     /// <summary>
     /// Dotted strings that indicate what the crumb is or where it comes from.

src/Sentry/Internal/PiiExtensions.cs

@@ -0,0 +1,53 @@
+namespace Sentry.Internal;
+
+/// <summary>
+/// Extensions to help redact data that might contain Personally Identifiable Information (PII) before sending it to
+/// Sentry.
+/// </summary>
+internal static class PiiExtensions
+{
+    internal const string RedactedText = "[Filtered]";
+    private static readonly Regex AuthRegex = new (@"(?i)\b(https?://.*@.*)\b", RegexOptions.Compiled);
+    private static readonly Regex UserInfoMatcher = new (@"^(?i)(https?://)(.*@)(.*)$", RegexOptions.Compiled);
+
+    /// <summary>
+    /// Searches for URLs in text data and redacts any PII data from these, as required.
+    /// </summary>
+    /// <param name="data">The data to be searched</param>
+    /// <returns>
+    /// The data, if no PII data is present or a copy of the data with PII data redacted otherwise
+    /// </returns>
+    public static string RedactUrl(this string data)
+    {
+        // If the data is empty then we don't need to redact anything
+        if (string.IsNullOrWhiteSpace(data))
+        {
+            return data;
+        }
+
+        // The pattern @"(?i)\b(https?://.*@.*)\b" uses the \b word boundary anchors to ensure that the match occurs at
+        // a word boundary. This allows the URL to be matched even if it is part of a larger text. The (?i) flag ensures
+        // case-insensitive matching for "https" or "http".
+        var result = AuthRegex.Replace(data, match =>
+        {
+            var matchedUrl = match.Groups[1].Value;
+            return RedactAuth(matchedUrl);
+        });
+
+        return result;
+    }
+
+    private static string RedactAuth(string data)
+    {
+        // ^ matches the start of the string. (?i)(https?://) gives a case-insensitive matching of the protocol.
+        // (.*@) matches the username and password (authentication information). (.*)$ matches the rest of the URL.
+        var match = UserInfoMatcher.Match(data);
+        if (match is not { Success: true, Groups.Count: 4 })
+        {
+            return data;
+        }
+        var userInfoString = match.Groups[2].Value;
+        var replacementString = userInfoString.Contains(":") ? "[Filtered]:[Filtered]@" : "[Filtered]@";
+        return match.Groups[1].Value + replacementString + match.Groups[3].Value;
+    }
+}

src/Sentry/SentryClient.cs

@@ -147,6 +147,11 @@ public void CaptureTransaction(Transaction transaction, Hint? hint)
             return;
         }
 
+        if (!_options.SendDefaultPii)
+        {
+            processedTransaction.Redact();
+        }
+
         CaptureEnvelope(Envelope.FromTransaction(processedTransaction));
     }
 
@@ -276,6 +281,11 @@ private SentryId DoSendEvent(SentryEvent @event, Hint? hint, Scope? scope)
             return SentryId.Empty;
         }
 
+        if (!_options.SendDefaultPii)
+        {
+            processedEvent.Redact();
+        }
+
         var attachments = hint.Attachments.ToList();
         var envelope = Envelope.FromEvent(processedEvent, _options.DiagnosticLogger, attachments, scope.SessionUpdate);
         return CaptureEnvelope(envelope) ? processedEvent.EventId : SentryId.Empty;

src/Sentry/SentryEvent.cs

@@ -242,6 +242,14 @@ public void SetTag(string key, string value) =>
     public void UnsetTag(string key) =>
         (_tags ??= new Dictionary<string, string>()).Remove(key);
 
+    internal void Redact()
+    {
+        foreach (var breadcrumb in Breadcrumbs)
+        {
+            breadcrumb.Redact();
+        }
+    }
+
     /// <inheritdoc />
     public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
     {

src/Sentry/SentryFailedRequestHandler.cs

@@ -79,24 +79,24 @@ public void HandleResponse(HttpResponseMessage response)
 
             var sentryRequest = new Request
             {
-                Url = uri?.AbsoluteUri,
                 QueryString = uri?.Query,
                 Method = response.RequestMessage.Method.Method,
             };
 
-            if (_options.SendDefaultPii)
-            {
-                sentryRequest.Cookies = response.RequestMessage.Headers.GetCookies();
-                sentryRequest.AddHeaders(response.RequestMessage.Headers);
-            }
-
             var responseContext = new Response {
                 StatusCode = (short)response.StatusCode,
                 BodySize = bodySize
             };
 
-            if (_options.SendDefaultPii)
+            if (!_options.SendDefaultPii)
             {
+                sentryRequest.Url = uri?.GetComponents(UriComponents.HttpRequestUrl, UriFormat.Unescaped);
+            }
+            else
+            {
+                sentryRequest.Url = uri?.AbsoluteUri;
+                sentryRequest.Cookies = response.RequestMessage.Headers.GetCookies();
+                sentryRequest.AddHeaders(response.RequestMessage.Headers);
                 responseContext.Cookies = response.Headers.GetCookies();
                 responseContext.AddHeaders(response.Headers);
             }

src/Sentry/Span.cs

@@ -1,4 +1,5 @@
 using Sentry.Extensibility;
+using Sentry.Internal;
 using Sentry.Internal.Extensions;
 
 namespace Sentry;
@@ -145,4 +146,9 @@ public static Span FromJson(JsonElement json)
             _extra = data!
         };
     }
+
+    internal void Redact()
+    {
+        Description = Description?.RedactUrl();
+    }
 }

src/Sentry/Transaction.cs

@@ -298,6 +298,23 @@ public void SetMeasurement(string name, Measurement measurement) =>
         SpanId,
         IsSampled);
 
+    /// <summary>
+    /// Redacts PII from the transaction
+    /// </summary>
+    internal void Redact()
+    {
+        Description = Description?.RedactUrl();
+        foreach (var breadcrumb in Breadcrumbs)
+        {
+            breadcrumb.Redact();
+        }
+
+        foreach (var span in Spans)
+        {
+            span.Redact();
+        }
+    }
+
     /// <inheritdoc />
     public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
     {

test/Directory.Build.props

@@ -39,6 +39,7 @@
     <Using Include="Sentry.Testing" />
 
     <Using Include="FluentAssertions" />
+    <Using Include="FluentAssertions.Execution" />
     <Using Include="NSubstitute" />
     <Using Include="NSubstitute.Core" />
     <Using Include="NSubstitute.ExceptionExtensions" />

test/Sentry.Tests/Internals/PiiExtensionsTests.cs

@@ -0,0 +1,38 @@
+namespace Sentry.Tests.Internals;
+
+public class PiiExtensionsTests
+{
+    [Fact]
+    public void RedactUrl_Null()
+    {
+        var actual = PiiExtensions.RedactUrl(null);
+
+        Assert.Null(actual);
+    }
+
+    [Theory]
+    [InlineData("I'm a harmless string.", "doesn't affect ordinary strings")]
+    [InlineData("htps://user:password@sentry.io?q=1&s=2&token=secret#top", "doesn't affect malformed https urls")]
+    [InlineData("htp://user:password@sentry.io?q=1&s=2&token=secret#top", "doesn't affect malformed http urls")]
+    public void RedactUrl_NotNull_WithoutPii(string original, string reason)
+    {
+        var actual = original.RedactUrl();
+
+        actual.Should().Be(original, reason);
+    }
+
+    [Theory]
+    [InlineData("https://user:password@sentry.io?q=1&s=2&token=secret#top", "https://[Filtered]:[Filtered]@sentry.io?q=1&s=2&token=secret#top", "strips user info with user and password from https")]
+    [InlineData("https://user:password@sentry.io", "https://[Filtered]:[Filtered]@sentry.io", "strips user info with user and password from https without query")]
+    [InlineData("https://user@sentry.io", "https://[Filtered]@sentry.io", "strips user info with user only from https without query")]
+    [InlineData("http://user:password@sentry.io?q=1&s=2&token=secret#top", "http://[Filtered]:[Filtered]@sentry.io?q=1&s=2&token=secret#top", "strips user info with user and password from http")]
+    [InlineData("http://user:password@sentry.io", "http://[Filtered]:[Filtered]@sentry.io", "strips user info with user and password from http without query")]
+    [InlineData("http://user@sentry.io", "http://[Filtered]@sentry.io", "strips user info with user only from http without query")]
+    [InlineData("GET https://user@sentry.io for goodness", "GET https://[Filtered]@sentry.io for goodness", "strips user info from URL embedded in text")]
+    public void RedactUrl_NotNull_WithPii(string original, string expected, string reason)
+    {
+        var actual = original.RedactUrl();
+
+        actual.Should().Be(expected, reason);
+    }
+}

test/Sentry.Tests/Internals/SentryScopeManagerTests.cs

@@ -1,4 +1,3 @@
-using FluentAssertions.Execution;
 using Sentry.Internal.ScopeStack;
 
 namespace Sentry.Tests.Internals;

test/Sentry.Tests/Protocol/BreadcrumbTests.cs

@@ -1,6 +1,6 @@
 namespace Sentry.Tests.Protocol;
 
-public class BreadcrumbTests : ImmutableTests<Breadcrumb>
+public class BreadcrumbTests
 {
     private readonly IDiagnosticLogger _testOutputLogger;
 
@@ -9,6 +9,50 @@ public BreadcrumbTests(ITestOutputHelper output)
         _testOutputLogger = new TestOutputDiagnosticLogger(output);
     }
 
+    [Fact]
+    public void Redact_Redacts_Urls()
+    {
+        // Arrange
+        var breadcrumbData = new Dictionary<string, string>
+        {
+            {"url", "https://user@sentry.io"},
+            {"method", "GET"},
+            {"status_code", "403"}
+        };
+        var timestamp = DateTimeOffset.UtcNow;
+        var message = "message https://user@sentry.io";
+        var type = "fake_type";
+        var data = breadcrumbData;
+        var category = "fake_category";
+        var level = BreadcrumbLevel.Error;
+
+        var breadcrumb = new Breadcrumb(
+            timestamp : timestamp,
+            message : message,
+            type : type,
+            data : breadcrumbData,
+            category : category,
+            level : level
+        );
+
+        // Act
+        breadcrumb.Redact();
+
+        // Assert
+        using (new AssertionScope())
+        {
+            breadcrumb.Should().NotBeNull();
+            breadcrumb.Timestamp.Should().Be(timestamp);
+            breadcrumb.Message.Should().Be("message https://[Filtered]@sentry.io"); // should be redacted
+            breadcrumb.Type.Should().Be(type);
+            breadcrumb.Data?["url"].Should().Be("https://[Filtered]@sentry.io"); // should be redacted
+            breadcrumb.Data?["method"].Should().Be(breadcrumb.Data?["method"]);
+            breadcrumb.Data?["status_code"].Should().Be(breadcrumb.Data?["status_code"]);
+            breadcrumb.Category.Should().Be(category);
+            breadcrumb.Level.Should().Be(level);
+        }
+    }
+
     [Fact]
     public void SerializeObject_ParameterlessConstructor_IncludesTimestamp()
     {