RUMM-1090 Logs Event Scrubbing

Datadog/Datadog.xcodeproj/project.pbxproj

@@ -47,9 +47,9 @@
 		61133BDF2423979B00786299 /* SwiftExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BBA2423979B00786299 /* SwiftExtensions.swift */; };
 		61133BE02423979B00786299 /* Datadog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BBB2423979B00786299 /* Datadog.swift */; };
 		61133BE32423979B00786299 /* UserInfoProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC02423979B00786299 /* UserInfoProvider.swift */; };
-		61133BE42423979B00786299 /* LogEncoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC22423979B00786299 /* LogEncoder.swift */; };
-		61133BE52423979B00786299 /* LogBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC32423979B00786299 /* LogBuilder.swift */; };
-		61133BE62423979B00786299 /* LogSanitizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC42423979B00786299 /* LogSanitizer.swift */; };
+		61133BE42423979B00786299 /* LogEventEncoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC22423979B00786299 /* LogEventEncoder.swift */; };
+		61133BE52423979B00786299 /* LogEventBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC32423979B00786299 /* LogEventBuilder.swift */; };
+		61133BE62423979B00786299 /* LogEventSanitizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC42423979B00786299 /* LogEventSanitizer.swift */; };
 		61133BE72423979B00786299 /* LogUtilityOutputs.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC62423979B00786299 /* LogUtilityOutputs.swift */; };
 		61133BE82423979B00786299 /* LogFileOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC72423979B00786299 /* LogFileOutput.swift */; };
 		61133BE92423979B00786299 /* LogOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61133BC82423979B00786299 /* LogOutput.swift */; };
@@ -508,6 +508,7 @@
 		B3FC3C3C2653A97700DEED9E /* VitalInfoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3FC3C3B2653A97700DEED9E /* VitalInfoTests.swift */; };
 		D2135330270CA722000315AD /* DataCompressionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D213532F270CA722000315AD /* DataCompressionTests.swift */; };
 		D24C27EA270C8BEE005DE596 /* DataCompression.swift in Sources */ = {isa = PBXBuildFile; fileRef = D24C27E9270C8BEE005DE596 /* DataCompression.swift */; };
+		D22C1F5C271484B400922024 /* LogEventMapper.swift in Sources */ = {isa = PBXBuildFile; fileRef = D22C1F5B271484B400922024 /* LogEventMapper.swift */; };
 		D2F1B81126D795F3009F3293 /* DDNoopRUMMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */; };
 		D2F1B81326D8DA68009F3293 /* DDNoopRUMMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */; };
 		D2F1B81526D8E5FF009F3293 /* DDNoopTracerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81426D8E5FF009F3293 /* DDNoopTracerTests.swift */; };
@@ -694,9 +695,9 @@
 		61133BBA2423979B00786299 /* SwiftExtensions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SwiftExtensions.swift; sourceTree = "<group>"; };
 		61133BBB2423979B00786299 /* Datadog.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Datadog.swift; sourceTree = "<group>"; };
 		61133BC02423979B00786299 /* UserInfoProvider.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UserInfoProvider.swift; sourceTree = "<group>"; };
-		61133BC22423979B00786299 /* LogEncoder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogEncoder.swift; sourceTree = "<group>"; };
-		61133BC32423979B00786299 /* LogBuilder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogBuilder.swift; sourceTree = "<group>"; };
-		61133BC42423979B00786299 /* LogSanitizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogSanitizer.swift; sourceTree = "<group>"; };
+		61133BC22423979B00786299 /* LogEventEncoder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogEventEncoder.swift; sourceTree = "<group>"; };
+		61133BC32423979B00786299 /* LogEventBuilder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogEventBuilder.swift; sourceTree = "<group>"; };
+		61133BC42423979B00786299 /* LogEventSanitizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogEventSanitizer.swift; sourceTree = "<group>"; };
 		61133BC62423979B00786299 /* LogUtilityOutputs.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogUtilityOutputs.swift; sourceTree = "<group>"; };
 		61133BC72423979B00786299 /* LogFileOutput.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogFileOutput.swift; sourceTree = "<group>"; };
 		61133BC82423979B00786299 /* LogOutput.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogOutput.swift; sourceTree = "<group>"; };
@@ -1157,6 +1158,7 @@
 		B3FC3C3B2653A97700DEED9E /* VitalInfoTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VitalInfoTests.swift; sourceTree = "<group>"; };
 		D213532F270CA722000315AD /* DataCompressionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataCompressionTests.swift; sourceTree = "<group>"; };
 		D24C27E9270C8BEE005DE596 /* DataCompression.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataCompression.swift; sourceTree = "<group>"; };
+		D22C1F5B271484B400922024 /* LogEventMapper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LogEventMapper.swift; sourceTree = "<group>"; };
 		D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitor.swift; sourceTree = "<group>"; };
 		D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitorTests.swift; sourceTree = "<group>"; };
 		D2F1B81426D8E5FF009F3293 /* DDNoopTracerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopTracerTests.swift; sourceTree = "<group>"; };
@@ -1493,6 +1495,7 @@
 				612983CC2449E62E00D4424B /* LoggingFeature.swift */,
 				61133BC12423979B00786299 /* Log */,
 				61133BC52423979B00786299 /* LogOutputs */,
+				D22C1F5A2714849700922024 /* Scrubbing */,
 			);
 			path = Logging;
 			sourceTree = "<group>";
@@ -1510,9 +1513,9 @@
 		61133BC12423979B00786299 /* Log */ = {
 			isa = PBXGroup;
 			children = (
-				61133BC22423979B00786299 /* LogEncoder.swift */,
-				61133BC32423979B00786299 /* LogBuilder.swift */,
-				61133BC42423979B00786299 /* LogSanitizer.swift */,
+				61133BC22423979B00786299 /* LogEventEncoder.swift */,
+				61133BC32423979B00786299 /* LogEventBuilder.swift */,
+				61133BC42423979B00786299 /* LogEventSanitizer.swift */,
 			);
 			path = Log;
 			sourceTree = "<group>";
@@ -3249,6 +3252,14 @@
 			path = RUMVitals;
 			sourceTree = "<group>";
 		};
+		D22C1F5A2714849700922024 /* Scrubbing */ = {
+			isa = PBXGroup;
+			children = (
+				D22C1F5B271484B400922024 /* LogEventMapper.swift */,
+			);
+			path = Scrubbing;
+			sourceTree = "<group>";
+		};
 /* End PBXGroup section */
 
 /* Begin PBXHeadersBuildPhase section */
@@ -3876,7 +3887,7 @@
 				616CCE13250A1868009FED46 /* RUMCommandSubscriber.swift in Sources */,
 				6112B11425C84E7900B37771 /* CrashReportingIntegration.swift in Sources */,
 				6161247925CA9CA6009901BE /* CrashReporter.swift in Sources */,
-				61133BE62423979B00786299 /* LogSanitizer.swift in Sources */,
+				61133BE62423979B00786299 /* LogEventSanitizer.swift in Sources */,
 				615F197C25B5A64B00BE14B5 /* UIKitExtensions.swift in Sources */,
 				9EC8B5DA2668197B000F7529 /* VitalCPUReader.swift in Sources */,
 				614D812C24E3EA15004C9C5D /* Feature.swift in Sources */,
@@ -3908,7 +3919,7 @@
 				D2F1B81126D795F3009F3293 /* DDNoopRUMMonitor.swift in Sources */,
 				61C5A88724509A0C00DA608C /* Casting.swift in Sources */,
 				61C3E63724BF191F008053F2 /* RUMScope.swift in Sources */,
-				61133BE52423979B00786299 /* LogBuilder.swift in Sources */,
+				61133BE52423979B00786299 /* LogEventBuilder.swift in Sources */,
 				61133BD42423979B00786299 /* FileReader.swift in Sources */,
 				9E359F4E26CD518D001E25E9 /* LongTaskObserver.swift in Sources */,
 				61C5A88A24509A0C00DA608C /* SpanFileOutput.swift in Sources */,
@@ -3930,14 +3941,15 @@
 				61B22E5A24F3E6B700DC26D2 /* RUMDebugging.swift in Sources */,
 				D24C27EA270C8BEE005DE596 /* DataCompression.swift in Sources */,
 				61C5A88B24509A0C00DA608C /* SpanOutput.swift in Sources */,
-				61133BE42423979B00786299 /* LogEncoder.swift in Sources */,
+				61133BE42423979B00786299 /* LogEventEncoder.swift in Sources */,
 				B3FC3C0926526F0000DEED9E /* VitalInfo.swift in Sources */,
 				613E81F025A740140084B751 /* RUMEventsMapper.swift in Sources */,
 				61D980BA24E28D0100E03345 /* RUMIntegrations.swift in Sources */,
 				61C5A88424509A0C00DA608C /* DDSpan.swift in Sources */,
 				61FF281E24B8968D000B3D9B /* RUMEventBuilder.swift in Sources */,
 				E1D5AEA724B4D45B007F194B /* Versioning.swift in Sources */,
 				61133BD82423979B00786299 /* HTTPClient.swift in Sources */,
+				D22C1F5C271484B400922024 /* LogEventMapper.swift in Sources */,
 				61B038542527246D00518F3C /* URLSessionSwizzler.swift in Sources */,
 				B3BBBCB2265E71C700943419 /* VitalMemoryReader.swift in Sources */,
 				61133BDB2423979B00786299 /* DatadogConfiguration.swift in Sources */,

Datadog/Example/Scenarios/Logging/LoggingScenarios.swift

@@ -4,7 +4,21 @@
  * Copyright 2019-2020 Datadog, Inc.
  */
 
+import Datadog
+
 /// Scenario which starts a view controller that sends bunch of logs to the server.
 final class LoggingManualInstrumentationScenario: TestScenario {
     static let storyboardName = "LoggingManualInstrumentationScenario"
+
+    func configureSDK(builder: Datadog.Configuration.Builder) {
+        _ = builder
+            .setLogEventMapper {
+                var log = $0
+                log.tags?.append("tag3:added")
+                if log.attributes.userAttributes["some-url"] != nil {
+                    log.attributes.userAttributes["some-url"] = "redacted"
+                }
+                return log
+            }
+    }
 }

Sources/Datadog/Core/Attributes/UserInfoProvider.swift

@@ -35,12 +35,24 @@ internal class UserInfoProvider {
     }
 }
 
-/// Information about the user.
 internal struct UserInfo {
-    let id: String?
-    let name: String?
-    let email: String?
-    let extraInfo: [AttributeKey: AttributeValue]
+    /// User ID, if any.
+    internal let id: String?
+    /// Name representing the user, if any.
+    internal let name: String?
+    /// User email, if any.
+    internal let email: String?
+    /// User custom attributes, if any.
+    internal var extraInfo: [AttributeKey: AttributeValue]
+}
 
-    internal static var empty: UserInfo { UserInfo(id: nil, name: nil, email: nil, extraInfo: [:]) }
+extension UserInfo {
+    internal static var empty: Self {
+        .init(
+            id: nil,
+            name: nil,
+            email: nil,
+            extraInfo: [:]
+        )
+    }
 }

Sources/Datadog/Core/FeaturesConfiguration.swift

@@ -26,6 +26,7 @@ internal struct FeaturesConfiguration {
         let common: Common
         let uploadURL: URL
         let clientToken: String
+        let logEventMapper: LogEventMapper?
     }
 
     struct Tracing {
@@ -168,7 +169,8 @@ extension FeaturesConfiguration {
             logging = Logging(
                 common: common,
                 uploadURL: try ifValid(endpointURLString: logsEndpoint.url),
-                clientToken: try ifValid(clientToken: configuration.clientToken)
+                clientToken: try ifValid(clientToken: configuration.clientToken),
+                logEventMapper: configuration.logEventMapper
             )
         }
 

Sources/Datadog/DatadogConfiguration.swift

@@ -249,6 +249,7 @@ extension Datadog {
 
         private(set) var serviceName: String?
         private(set) var firstPartyHosts: Set<String>?
+        private(set) var logEventMapper: LogEventMapper?
         private(set) var spanEventMapper: SpanEventMapper?
         private(set) var rumSessionsSamplingRate: Float
         private(set) var rumSessionsListener: RUMSessionListener?
@@ -401,6 +402,15 @@ extension Datadog {
                 return self
             }
 
+            /// Sets the custom mapper for `LogEvent`. This can be used to modify logs before they are send to Datadog.
+            /// - Parameter mapper: the closure taking `LogEvent` as input and expecting `LogEvent` as output.
+            /// The implementation should obtain a mutable version of the `LogEvent`, modify it and return it. Returning `nil` will result
+            /// with dropping the Log event entirely, so it won't be send to Datadog.
+            public func setLogEventMapper(_ mapper: @escaping (LogEvent) -> LogEvent) -> Builder {
+                configuration.logEventMapper = mapper
+                return self
+            }
+
             /// Sets the server endpoint to which logs are sent.
             /// - Parameter logsEndpoint: server endpoint (default value is `LogsEndpoint.us`)
             @available(*, deprecated, message: "This option is replaced by `set(endpoint:)`. Refer to the new API comment for details.")
@@ -483,7 +493,7 @@ extension Datadog {
             ///
             /// **NOTE** The mapper intentionally prevents from returning a `nil` to drop the `SpanEvent` entirely, this ensures that all spans are sent to Datadog.
             ///
-            /// Use the `trackURLSession(firstPartyHosts:)` API to confiture tracing only the hosts that you are interested in.
+            /// Use the `trackURLSession(firstPartyHosts:)` API to configure tracing only the hosts that you are interested in.
             public func setSpanEventMapper(_ mapper: @escaping (SpanEvent) -> SpanEvent) -> Builder {
                 configuration.spanEventMapper = mapper
                 return self

Sources/Datadog/FeaturesIntegration/CrashReporting/CrashReportingWithLoggingIntegration.swift

@@ -67,19 +67,21 @@ internal struct CrashReportingWithLoggingIntegration: CrashReportingIntegration
 
     // MARK: - Building Log
 
-    private func createLog(from crashReport: DDCrashReport, crashContext: CrashContext, crashDate: Date) -> Log {
+    private func createLog(from crashReport: DDCrashReport, crashContext: CrashContext, crashDate: Date) -> LogEvent {
         var errorAttributes: [AttributeKey: AttributeValue] = [:]
         errorAttributes[DDError.threads] = crashReport.threads
         errorAttributes[DDError.binaryImages] = crashReport.binaryImages
         errorAttributes[DDError.meta] = crashReport.meta
         errorAttributes[DDError.wasTruncated] = crashReport.wasTruncated
 
-        return Log(
+        let user = crashContext.lastUserInfo
+
+        return LogEvent(
             date: crashDate,
             status: .emergency,
             message: crashReport.message,
-            error: DDError(
-                type: crashReport.type,
+            error: .init(
+                kind: crashReport.type,
                 message: crashReport.message,
                 stack: crashReport.stack
             ),
@@ -89,7 +91,12 @@ internal struct CrashReportingWithLoggingIntegration: CrashReportingIntegration
             loggerVersion: sdkVersion,
             threadName: nil,
             applicationVersion: configuration.applicationVersion,
-            userInfo: crashContext.lastUserInfo ?? .empty,
+            userInfo: .init(
+                id: user?.id,
+                name: user?.name,
+                email: user?.email,
+                extraInfo: user?.extraInfo ?? [:]
+            ),
             networkConnectionInfo: crashContext.lastNetworkConnectionInfo,
             mobileCarrierInfo: crashContext.lastCarrierInfo,
             attributes: .init(userAttributes: [:], internalAttributes: errorAttributes),

Sources/Datadog/FeaturesIntegration/LoggingForTracingAdapter.swift

@@ -18,15 +18,16 @@ internal struct LoggingForTracingAdapter {
 
     func resolveLogOutput(usingTracingFeature tracingFeature: TracingFeature, tracerConfiguration: Tracer.Configuration) -> AdaptedLogOutput {
         return AdaptedLogOutput(
-            logBuilder: LogBuilder(
+            logBuilder: LogEventBuilder(
                 applicationVersion: tracingFeature.configuration.common.applicationVersion,
                 environment: tracingFeature.configuration.common.environment,
                 serviceName: tracerConfiguration.serviceName ?? tracingFeature.configuration.common.serviceName,
                 loggerName: "trace",
                 userInfoProvider: tracingFeature.userInfoProvider,
                 networkConnectionInfoProvider: tracerConfiguration.sendNetworkInfo ? tracingFeature.networkConnectionInfoProvider : nil,
                 carrierInfoProvider: tracerConfiguration.sendNetworkInfo ? tracingFeature.carrierInfoProvider : nil,
-                dateCorrector: loggingFeature.dateCorrector
+                dateCorrector: loggingFeature.dateCorrector,
+                logEventMapper: loggingFeature.configuration.logEventMapper
             ),
             loggingOutput: LogFileOutput(
                 fileWriter: loggingFeature.storage.writer,
@@ -52,7 +53,7 @@ internal struct LoggingForTracingAdapter {
         }
 
         /// Log builder using Tracing configuration.
-        let logBuilder: LogBuilder
+        let logBuilder: LogEventBuilder
         /// Actual `LogOutput` bridged to `LoggingFeature`.
         let loggingOutput: LogOutput
 
@@ -89,13 +90,16 @@ internal struct LoggingForTracingAdapter {
                 message: message,
                 error: extractedError,
                 date: date,
-                attributes: LogAttributes(
+                attributes: LogEvent.Attributes(
                     userAttributes: userAttributes,
                     internalAttributes: internalAttributes
                 ),
                 tags: []
             )
-            loggingOutput.write(log: log)
+
+            if let event = log {
+                loggingOutput.write(log: event)
+            }
         }
     }
 }

Sources/Datadog/FeaturesIntegration/LoggingWithRUMIntegration.swift

@@ -26,10 +26,10 @@ internal struct LoggingWithRUMContextIntegration {
 internal struct LoggingWithRUMErrorsIntegration {
     private let rumErrorsIntegration = RUMErrorsIntegration()
 
-    func addError(for log: Log) {
+    func addError(for log: LogEvent) {
         rumErrorsIntegration.addError(
             with: log.error?.message ?? log.message,
-            type: log.error?.type,
+            type: log.error?.kind,
             stack: log.error?.stack,
             source: .logger
         )

Sources/Datadog/InternalMonitoring/InternalMonitoringFeature.swift

@@ -121,15 +121,16 @@ internal final class InternalMonitoringFeature {
 
         // Initialize internal monitor
         let internalLogger = Logger(
-            logBuilder: LogBuilder(
+            logBuilder: LogEventBuilder(
                 applicationVersion: configuration.common.applicationVersion,
                 environment: configuration.sdkEnvironment,
                 serviceName: configuration.sdkServiceName,
                 loggerName: configuration.loggerName,
                 userInfoProvider: UserInfoProvider(), // no-op to not associate user info with internal logs
                 networkConnectionInfoProvider: commonDependencies.networkConnectionInfoProvider,
                 carrierInfoProvider: commonDependencies.carrierInfoProvider,
-                dateCorrector: commonDependencies.dateCorrector
+                dateCorrector: commonDependencies.dateCorrector,
+                logEventMapper: nil
             ),
             logOutput: LogFileOutput(
                 fileWriter: storage.writer,