Merge pull request #797 from DataDog/maxep/RUMM-1870/data-encryption

RUMM-1870 Add data encryption interface

CHANGELOG.md

@@ -3,6 +3,7 @@
 * [FEATURE] Web-view tracking. See [#729][]
 * [FEATURE] Integration with CI Visibility Tests. See[#761][]
 * [FEATURE] Add tvOS Support. See [#793][]
+* [FEATURE] Add data encryption interface on-disk data storage. See [#797][]
 * [BUGFIX] Strip query parameters from span resource. See [#728][]
 * [BUGFIX] Stop reporting pre-warmed application launch time. See [#789][]
 * [BUGFIX] Allow log event dropping. See [#795][]
@@ -334,6 +335,7 @@
 [#793]: https://github.com/DataDog/dd-sdk-ios/issues/793
 [#794]: https://github.com/DataDog/dd-sdk-ios/issues/794
 [#795]: https://github.com/DataDog/dd-sdk-ios/issues/795
+[#797]: https://github.com/DataDog/dd-sdk-ios/pull/797
 [@00FA9A]: https://github.com/00FA9A
 [@Britton-Earnin]: https://github.com/Britton-Earnin
 [@Hengyu]: https://github.com/Hengyu

Datadog/Datadog.xcodeproj/project.pbxproj

@@ -981,6 +981,8 @@
 		D2CB6FF327C5369600A62B57 /* DatadogCrashReporting.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D2CB6FD127C5348200A62B57 /* DatadogCrashReporting.framework */; };
 		D2DC4BBC27F234D600E4FB96 /* CITestIntegration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E11625D727B681D200E428C6 /* CITestIntegration.swift */; };
 		D2DC4BBD27F234E000E4FB96 /* CITestIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E143CCAE27D236F600F4018A /* CITestIntegrationTests.swift */; };
+		D2DC4BF627F484AA00E4FB96 /* DataEncryption.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */; };
+		D2DC4BF727F484AA00E4FB96 /* DataEncryption.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */; };
 		D2EFF3D32731822A00D09F33 /* RUMViewsHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2EFF3D22731822A00D09F33 /* RUMViewsHandler.swift */; };
 		D2F1B81126D795F3009F3293 /* DDNoopRUMMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */; };
 		D2F1B81326D8DA68009F3293 /* DDNoopRUMMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */; };
@@ -1736,6 +1738,7 @@
 		D2CB6FB027C5217A00A62B57 /* DatadogObjc.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = DatadogObjc.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		D2CB6FD127C5348200A62B57 /* DatadogCrashReporting.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = DatadogCrashReporting.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		D2CB6FEC27C5352300A62B57 /* DatadogCrashReportingTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = DatadogCrashReportingTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
+		D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DataEncryption.swift; sourceTree = "<group>"; };
 		D2EFF3D22731822A00D09F33 /* RUMViewsHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMViewsHandler.swift; sourceTree = "<group>"; };
 		D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitor.swift; sourceTree = "<group>"; };
 		D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitorTests.swift; sourceTree = "<group>"; };
@@ -2098,6 +2101,7 @@
 			children = (
 				61AD4E3724531500006E34EA /* DataFormat.swift */,
 				6137C571271DAD4B00EFC4A1 /* DataOrchestrator.swift */,
+				D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */,
 				61133BA92423979B00786299 /* FilesOrchestrator.swift */,
 				613E79412577C08900DFCC17 /* Writing */,
 				613E79422577C09B00DFCC17 /* Reading */,
@@ -5020,6 +5024,7 @@
 				61133BDF2423979B00786299 /* SwiftExtensions.swift in Sources */,
 				61D3E0D3277B23F1008BE766 /* KronosDNSResolver.swift in Sources */,
 				6149FB3A2529D17F00EE387A /* InternalURLsFilter.swift in Sources */,
+				D2DC4BF627F484AA00E4FB96 /* DataEncryption.swift in Sources */,
 				611529A525E3DD51004F740E /* ValuePublisher.swift in Sources */,
 				61FFFB89278457D400401A28 /* KronosMonitor.swift in Sources */,
 				618DCFD724C7265300589570 /* RUMUUID.swift in Sources */,
@@ -5590,6 +5595,7 @@
 				D2CB6E4427C50EAE00A62B57 /* UIApplicationSwizzler.swift in Sources */,
 				D2CB6E4527C50EAE00A62B57 /* RUMResourceScope.swift in Sources */,
 				D2CB6E4627C50EAE00A62B57 /* RUMSessionScope.swift in Sources */,
+				D2DC4BF727F484AA00E4FB96 /* DataEncryption.swift in Sources */,
 				D2CB6E4727C50EAE00A62B57 /* TracingUUID.swift in Sources */,
 				D2CB6E4827C50EAE00A62B57 /* ServerDateProvider.swift in Sources */,
 				D2CB6E4927C50EAE00A62B57 /* AttributesSanitizer.swift in Sources */,

Sources/Datadog/Core/Feature.swift

@@ -32,6 +32,7 @@ internal struct FeaturesCommonDependencies {
     let carrierInfoProvider: CarrierInfoProviderType
     let launchTimeProvider: LaunchTimeProviderType
     let appStateListener: AppStateListening
+    let encryption: DataEncryption?
 }
 
 internal struct FeatureStorage {
@@ -82,12 +83,14 @@ internal struct FeatureStorage {
         let unauthorizedFileWriter = FileWriter(
             dataFormat: dataFormat,
             orchestrator: unauthorizedFilesOrchestrator,
+            encryption: commonDependencies.encryption,
             internalMonitor: internalMonitor
         )
 
         let authorizedFileWriter = FileWriter(
             dataFormat: dataFormat,
             orchestrator: authorizedFilesOrchestrator,
+            encryption: commonDependencies.encryption,
             internalMonitor: internalMonitor
         )
 
@@ -116,6 +119,7 @@ internal struct FeatureStorage {
             fileReader: FileReader(
                 dataFormat: dataFormat,
                 orchestrator: authorizedFilesOrchestrator,
+                encryption: commonDependencies.encryption,
                 internalMonitor: internalMonitor
             )
         )

Sources/Datadog/Core/FeaturesConfiguration.swift

@@ -22,6 +22,7 @@ internal struct FeaturesConfiguration {
         let origin: String?
         let sdkVersion: String
         let proxyConfiguration: [AnyHashable: Any]?
+        let encryption: DataEncryption?
     }
 
     struct Logging {
@@ -174,7 +175,8 @@ extension FeaturesConfiguration {
             source: source,
             origin: CITestIntegration.active?.origin,
             sdkVersion: sdkVersion,
-            proxyConfiguration: configuration.proxyConfiguration
+            proxyConfiguration: configuration.proxyConfiguration,
+            encryption: configuration.encryption
         )
 
         if configuration.loggingEnabled {

Sources/Datadog/Core/Persistence/DataEncryption.swift

@@ -0,0 +1,29 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-2022 Datadog, Inc.
+ */
+
+import Foundation
+
+/// Interface that allows storing data in encrypted format. Encryption/decryption round should
+/// return exactly the same data as it given for the encryption originally (even if decryption
+/// happens in another process/app launch).
+public protocol DataEncryption {
+    /// Encrypts given `Data` with user-chosen encryption.
+    ///
+    /// - Parameter data: Data to encrypt.
+    /// - Returns: The encrypted data.
+    func encrypt(data: Data) throws -> Data
+
+    /// Decrypts given `Data` with user-chosen encryption.
+    ///
+    /// Beware that data to decrypt could be encrypted in a previous app launch, so
+    /// implementation should be aware of the case when decryption could fail (for example,
+    /// key used for encryption is different from key used for decryption, if they are unique
+    /// for every app launch).
+    ///
+    /// - Parameter data: Data to decrypt.
+    /// - Returns: The decrypted data.
+    func decrypt(data: Data) throws -> Data
+}

Sources/Datadog/Core/Persistence/DataFormat.swift

@@ -13,17 +13,17 @@ internal struct DataFormat {
     /// Suffixes the batch payload read from file.
     let suffixData: Data
     /// Separates entities written to file.
-    let separatorData: Data
+    let separatorByte: UInt8
 
     // MARK: - Initialization
 
     init(
         prefix: String,
         suffix: String,
-        separator: String
+        separator: Character
     ) {
         self.prefixData = prefix.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
         self.suffixData = suffix.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
-        self.separatorData = separator.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
+        self.separatorByte = separator.asciiValue!   // swiftlint:disable:this force_unwrapping
     }
 }

Sources/Datadog/Core/Persistence/Reading/FileReader.swift

@@ -12,42 +12,91 @@ internal final class FileReader: Reader {
     private let dataFormat: DataFormat
     /// Orchestrator producing reference to readable file.
     private let orchestrator: FilesOrchestrator
+    private let encryption: DataEncryption?
     private let internalMonitor: InternalMonitor?
 
     /// Files marked as read.
-    private var filesRead: [ReadableFile] = []
+    private var filesRead: Set<String> = []
 
     init(
         dataFormat: DataFormat,
         orchestrator: FilesOrchestrator,
+        encryption: DataEncryption? = nil,
         internalMonitor: InternalMonitor? = nil
     ) {
         self.dataFormat = dataFormat
         self.orchestrator = orchestrator
+        self.encryption = encryption
         self.internalMonitor = internalMonitor
     }
 
     // MARK: - Reading batches
 
     func readNextBatch() -> Batch? {
-        if let file = orchestrator.getReadableFile(excludingFilesNamed: Set(filesRead.map { $0.name })) {
-            do {
-                let fileData = try file.read()
-                let batchData = dataFormat.prefixData + fileData + dataFormat.suffixData
-                return Batch(data: batchData, file: file)
-            } catch {
-                internalMonitor?.sdkLogger.error("Failed to read data from file", error: error)
-                return nil
-            }
+        guard let file = orchestrator.getReadableFile(excludingFilesNamed: filesRead) else {
+            return nil
+        }
+
+        do {
+            let fileData = try decrypt(data: file.read())
+            let batchData = dataFormat.prefixData + fileData + dataFormat.suffixData
+            return Batch(data: batchData, file: file)
+        } catch {
+            internalMonitor?.sdkLogger.error("Failed to read data from file", error: error)
+            return nil
+        }
+    }
+
+    /// Decrypts data if encryption is available.
+    ///
+    /// When encryption is provided, the data is splitted using data-format separator, each slices
+    /// is then decoded from base64 and decrypted. Data is finally re-joined with data-format separator.
+    ///
+    /// If no encryption, the data is returned.
+    ///
+    /// - Parameter data: The data to decrypt.
+    /// - Returns: Decrypted data.
+    private func decrypt(data: Data) -> Data {
+        guard let encryption = encryption else {
+            return data
         }
 
-        return nil
+        var failure: String? = nil
+        defer {
+            failure.map { userLogger.error($0) }
+        }
+
+        return data
+            // split data
+            .split(separator: dataFormat.separatorByte)
+            // decode base64 - report failure
+            .compactMap {
+                if let data = Data(base64Encoded: $0) {
+                    return data
+                }
+
+                failure = "🔥 Failed to decode base64 data before decryption"
+                return nil
+            }
+            // decrypt data - report failure
+            .compactMap { (data: Data) in
+                do {
+                    return try encryption.decrypt(data: data)
+                } catch {
+                    failure = "🔥 Failed to decrypt data with error: \(error)"
+                    return nil
+                }
+            }
+            // concat data
+            .reduce(Data()) { $0 + $1 + [dataFormat.separatorByte] }
+            // drop last separator
+            .dropLast()
     }
 
     // MARK: - Accepting batches
 
     func markBatchAsRead(_ batch: Batch) {
         orchestrator.delete(readableFile: batch.file)
-        filesRead.append(batch.file)
+        filesRead.insert(batch.file.name)
     }
 }

Sources/Datadog/Core/Persistence/Writing/FileWriter.swift

@@ -14,16 +14,19 @@ internal final class FileWriter: Writer {
     private let orchestrator: FilesOrchestrator
     /// JSON encoder used to encode data.
     private let jsonEncoder: JSONEncoder
+    private let encryption: DataEncryption?
     private let internalMonitor: InternalMonitor?
 
     init(
         dataFormat: DataFormat,
         orchestrator: FilesOrchestrator,
+        encryption: DataEncryption? = nil,
         internalMonitor: InternalMonitor? = nil
     ) {
         self.dataFormat = dataFormat
         self.orchestrator = orchestrator
-        self.jsonEncoder = JSONEncoder.default()
+        self.jsonEncoder = .default()
+        self.encryption = encryption
         self.internalMonitor = internalMonitor
     }
 
@@ -32,18 +35,34 @@ internal final class FileWriter: Writer {
     /// Encodes given value to JSON data and writes it to the file.
     func write<T: Encodable>(value: T) {
         do {
-            let data = try jsonEncoder.encode(value)
+            var data = try encode(value: value)
             let file = try orchestrator.getWritableFile(writeSize: UInt64(data.count))
 
-            if try file.size() == 0 {
-                try file.append(data: data)
-            } else {
-                let atomicData = dataFormat.separatorData + data
-                try file.append(data: atomicData)
+            if try file.size() > 0 {
+                data.insert(dataFormat.separatorByte, at: 0)
             }
+
+            try file.append(data: data)
         } catch {
             userLogger.error("🔥 Failed to write data: \(error)")
             internalMonitor?.sdkLogger.error("Failed to write data to file", error: error)
         }
     }
+
+    /// Encodes the given encodable value and encrypt it if encryption is available.
+    ///
+    /// If encryption is available, encryption result is base64 encoded.
+    ///
+    /// - Parameter value: The value to encode.
+    /// - Returns: Data representation of the value.
+    private func encode<T: Encodable>(value: T) throws -> Data {
+        let data = try jsonEncoder.encode(value)
+
+        guard let encryption = encryption else {
+            return data
+        }
+
+        return try encryption.encrypt(data: data)
+            .base64EncodedData()
+    }
 }

Sources/Datadog/Datadog.swift

@@ -227,7 +227,8 @@ public class Datadog {
             networkConnectionInfoProvider: networkConnectionInfoProvider,
             carrierInfoProvider: carrierInfoProvider,
             launchTimeProvider: launchTimeProvider,
-            appStateListener: AppStateListener(dateProvider: dateProvider)
+            appStateListener: AppStateListener(dateProvider: dateProvider),
+            encryption: configuration.common.encryption
         )
 
         if let internalMonitoringConfiguration = configuration.internalMonitoring {

Sources/Datadog/DatadogConfiguration.swift

@@ -267,6 +267,7 @@ extension Datadog {
         private(set) var uploadFrequency: UploadFrequency
         private(set) var additionalConfiguration: [String: Any]
         private(set) var proxyConfiguration: [AnyHashable: Any]?
+        private(set) var encryption: DataEncryption?
 
         /// The client token autorizing internal monitoring data to be sent to Datadog org.
         private(set) var internalMonitoringClientToken: String?
@@ -770,6 +771,13 @@ extension Datadog {
                 return self
             }
 
+            /// Sets data encryption to use for on-disk data persistency.
+            /// - Parameter encryption: An encryption object complying with `DataEncryption` protocol.
+            public func set(encryption: DataEncryption) -> Builder {
+                configuration.encryption = encryption
+                return self
+            }
+
             /// Builds `Datadog.Configuration` object.
             public func build() -> Configuration {
                 return configuration