Dogfood recent changes

CHANGELOG.md

@@ -3,6 +3,7 @@
 * [FEATURE] Web-view tracking. See [#729][]
 * [FEATURE] Integration with CI Visibility Tests. See[#761][]
 * [FEATURE] Add tvOS Support. See [#793][]
+* [FEATURE] Add data encryption interface on-disk data storage. See [#797][]
 * [BUGFIX] Strip query parameters from span resource. See [#728][]
 * [BUGFIX] Stop reporting pre-warmed application launch time. See [#789][]
 * [BUGFIX] Allow log event dropping. See [#795][]
@@ -334,6 +335,7 @@
 [#793]: https://github.com/DataDog/dd-sdk-ios/issues/793
 [#794]: https://github.com/DataDog/dd-sdk-ios/issues/794
 [#795]: https://github.com/DataDog/dd-sdk-ios/issues/795
+[#797]: https://github.com/DataDog/dd-sdk-ios/pull/797
 [@00FA9A]: https://github.com/00FA9A
 [@Britton-Earnin]: https://github.com/Britton-Earnin
 [@Hengyu]: https://github.com/Hengyu

Datadog/Datadog.xcodeproj/project.pbxproj

@@ -981,6 +981,8 @@
 		D2CB6FF327C5369600A62B57 /* DatadogCrashReporting.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D2CB6FD127C5348200A62B57 /* DatadogCrashReporting.framework */; };
 		D2DC4BBC27F234D600E4FB96 /* CITestIntegration.swift in Sources */ = {isa = PBXBuildFile; fileRef = E11625D727B681D200E428C6 /* CITestIntegration.swift */; };
 		D2DC4BBD27F234E000E4FB96 /* CITestIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E143CCAE27D236F600F4018A /* CITestIntegrationTests.swift */; };
+		D2DC4BF627F484AA00E4FB96 /* DataEncryption.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */; };
+		D2DC4BF727F484AA00E4FB96 /* DataEncryption.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */; };
 		D2EFF3D32731822A00D09F33 /* RUMViewsHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2EFF3D22731822A00D09F33 /* RUMViewsHandler.swift */; };
 		D2F1B81126D795F3009F3293 /* DDNoopRUMMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */; };
 		D2F1B81326D8DA68009F3293 /* DDNoopRUMMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */; };
@@ -1736,6 +1738,7 @@
 		D2CB6FB027C5217A00A62B57 /* DatadogObjc.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = DatadogObjc.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		D2CB6FD127C5348200A62B57 /* DatadogCrashReporting.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = DatadogCrashReporting.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		D2CB6FEC27C5352300A62B57 /* DatadogCrashReportingTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = DatadogCrashReportingTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
+		D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DataEncryption.swift; sourceTree = "<group>"; };
 		D2EFF3D22731822A00D09F33 /* RUMViewsHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMViewsHandler.swift; sourceTree = "<group>"; };
 		D2F1B81026D795F3009F3293 /* DDNoopRUMMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitor.swift; sourceTree = "<group>"; };
 		D2F1B81226D8DA68009F3293 /* DDNoopRUMMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNoopRUMMonitorTests.swift; sourceTree = "<group>"; };
@@ -2098,6 +2101,7 @@
 			children = (
 				61AD4E3724531500006E34EA /* DataFormat.swift */,
 				6137C571271DAD4B00EFC4A1 /* DataOrchestrator.swift */,
+				D2DC4BF527F484AA00E4FB96 /* DataEncryption.swift */,
 				61133BA92423979B00786299 /* FilesOrchestrator.swift */,
 				613E79412577C08900DFCC17 /* Writing */,
 				613E79422577C09B00DFCC17 /* Reading */,
@@ -5020,6 +5024,7 @@
 				61133BDF2423979B00786299 /* SwiftExtensions.swift in Sources */,
 				61D3E0D3277B23F1008BE766 /* KronosDNSResolver.swift in Sources */,
 				6149FB3A2529D17F00EE387A /* InternalURLsFilter.swift in Sources */,
+				D2DC4BF627F484AA00E4FB96 /* DataEncryption.swift in Sources */,
 				611529A525E3DD51004F740E /* ValuePublisher.swift in Sources */,
 				61FFFB89278457D400401A28 /* KronosMonitor.swift in Sources */,
 				618DCFD724C7265300589570 /* RUMUUID.swift in Sources */,
@@ -5590,6 +5595,7 @@
 				D2CB6E4427C50EAE00A62B57 /* UIApplicationSwizzler.swift in Sources */,
 				D2CB6E4527C50EAE00A62B57 /* RUMResourceScope.swift in Sources */,
 				D2CB6E4627C50EAE00A62B57 /* RUMSessionScope.swift in Sources */,
+				D2DC4BF727F484AA00E4FB96 /* DataEncryption.swift in Sources */,
 				D2CB6E4727C50EAE00A62B57 /* TracingUUID.swift in Sources */,
 				D2CB6E4827C50EAE00A62B57 /* ServerDateProvider.swift in Sources */,
 				D2CB6E4927C50EAE00A62B57 /* AttributesSanitizer.swift in Sources */,

Datadog/Datadog.xcodeproj/xcshareddata/xcschemes/Datadog tvOS.xcscheme

@@ -30,6 +30,15 @@
       enableThreadSanitizer = "YES"
       codeCoverageEnabled = "YES"
       onlyGenerateCoverageForSpecifiedTargets = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "D2CB6ED327C520D400A62B57"
+            BuildableName = "DatadogTests.xctest"
+            BlueprintName = "DatadogTests tvOS"
+            ReferencedContainer = "container:Datadog.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
       <CommandLineArguments>
          <CommandLineArgument
             argument = "IS_RUNNING_UNIT_TESTS"

Datadog/Datadog.xcodeproj/xcshareddata/xcschemes/DatadogCrashReporting tvOS.xcscheme

@@ -30,6 +30,15 @@
       enableThreadSanitizer = "YES"
       codeCoverageEnabled = "YES"
       onlyGenerateCoverageForSpecifiedTargets = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "D2CB6FD327C5352300A62B57"
+            BuildableName = "DatadogCrashReportingTests.xctest"
+            BlueprintName = "DatadogCrashReportingTests tvOS"
+            ReferencedContainer = "container:Datadog.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
       <CommandLineArguments>
          <CommandLineArgument
             argument = "IS_RUNNING_UNIT_TESTS"

Datadog/Datadog.xcodeproj/xcshareddata/xcschemes/DatadogObjc tvOS.xcscheme

@@ -43,6 +43,15 @@
       shouldUseLaunchSchemeArgsEnv = "NO"
       codeCoverageEnabled = "YES"
       onlyGenerateCoverageForSpecifiedTargets = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "D2CB6ED327C520D400A62B57"
+            BuildableName = "DatadogTests.xctest"
+            BlueprintName = "DatadogTests tvOS"
+            ReferencedContainer = "container:Datadog.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
       <CommandLineArguments>
          <CommandLineArgument
             argument = "IS_RUNNING_UNIT_TESTS"

Makefile

@@ -4,8 +4,10 @@ all: dependencies xcodeproj-httpservermock templates
 DD_SDK_SWIFT_TESTING_VERSION = 1.1.1
 
 define DD_SDK_TESTING_XCCONFIG_CI
-FRAMEWORK_SEARCH_PATHS=$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/**\n
-LD_RUNPATH_SEARCH_PATHS=$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/**\n
+FRAMEWORK_SEARCH_PATHS[sdk=iphonesimulator*]=$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/ios-arm64_x86_64-simulator/\n
+LD_RUNPATH_SEARCH_PATHS[sdk=iphonesimulator*]=$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/ios-arm64_x86_64-simulator/\n
+FRAMEWORK_SEARCH_PATHS[sdk=appletvsimulator*]==$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/tvos-arm64_x86_64-simulator/\n
+LD_RUNPATH_SEARCH_PATHS[sdk=appletvsimulator*]=$$(inherited) $$(SRCROOT)/../instrumented-tests/DatadogSDKTesting.xcframework/tvos-arm64_x86_64-simulator/\n
 OTHER_LDFLAGS=$$(inherited) -framework DatadogSDKTesting\n
 DD_TEST_RUNNER=1\n
 DD_SDK_SWIFT_TESTING_SERVICE=dd-sdk-ios\n

Sources/Datadog/Core/Feature.swift

@@ -32,6 +32,7 @@ internal struct FeaturesCommonDependencies {
     let carrierInfoProvider: CarrierInfoProviderType
     let launchTimeProvider: LaunchTimeProviderType
     let appStateListener: AppStateListening
+    let encryption: DataEncryption?
 }
 
 internal struct FeatureStorage {
@@ -82,12 +83,14 @@ internal struct FeatureStorage {
         let unauthorizedFileWriter = FileWriter(
             dataFormat: dataFormat,
             orchestrator: unauthorizedFilesOrchestrator,
+            encryption: commonDependencies.encryption,
             internalMonitor: internalMonitor
         )
 
         let authorizedFileWriter = FileWriter(
             dataFormat: dataFormat,
             orchestrator: authorizedFilesOrchestrator,
+            encryption: commonDependencies.encryption,
             internalMonitor: internalMonitor
         )
 
@@ -116,6 +119,7 @@ internal struct FeatureStorage {
             fileReader: FileReader(
                 dataFormat: dataFormat,
                 orchestrator: authorizedFilesOrchestrator,
+                encryption: commonDependencies.encryption,
                 internalMonitor: internalMonitor
             )
         )

Sources/Datadog/Core/FeaturesConfiguration.swift

@@ -22,6 +22,7 @@ internal struct FeaturesConfiguration {
         let origin: String?
         let sdkVersion: String
         let proxyConfiguration: [AnyHashable: Any]?
+        let encryption: DataEncryption?
     }
 
     struct Logging {
@@ -174,7 +175,8 @@ extension FeaturesConfiguration {
             source: source,
             origin: CITestIntegration.active?.origin,
             sdkVersion: sdkVersion,
-            proxyConfiguration: configuration.proxyConfiguration
+            proxyConfiguration: configuration.proxyConfiguration,
+            encryption: configuration.encryption
         )
 
         if configuration.loggingEnabled {

Sources/Datadog/Core/Persistence/DataEncryption.swift

@@ -0,0 +1,29 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-2022 Datadog, Inc.
+ */
+
+import Foundation
+
+/// Interface that allows storing data in encrypted format. Encryption/decryption round should
+/// return exactly the same data as it given for the encryption originally (even if decryption
+/// happens in another process/app launch).
+public protocol DataEncryption {
+    /// Encrypts given `Data` with user-chosen encryption.
+    ///
+    /// - Parameter data: Data to encrypt.
+    /// - Returns: The encrypted data.
+    func encrypt(data: Data) throws -> Data
+
+    /// Decrypts given `Data` with user-chosen encryption.
+    ///
+    /// Beware that data to decrypt could be encrypted in a previous app launch, so
+    /// implementation should be aware of the case when decryption could fail (for example,
+    /// key used for encryption is different from key used for decryption, if they are unique
+    /// for every app launch).
+    ///
+    /// - Parameter data: Data to decrypt.
+    /// - Returns: The decrypted data.
+    func decrypt(data: Data) throws -> Data
+}

Sources/Datadog/Core/Persistence/DataFormat.swift

@@ -13,17 +13,17 @@ internal struct DataFormat {
     /// Suffixes the batch payload read from file.
     let suffixData: Data
     /// Separates entities written to file.
-    let separatorData: Data
+    let separatorByte: UInt8
 
     // MARK: - Initialization
 
     init(
         prefix: String,
         suffix: String,
-        separator: String
+        separator: Character
     ) {
         self.prefixData = prefix.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
         self.suffixData = suffix.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
-        self.separatorData = separator.data(using: .utf8)! // swiftlint:disable:this force_unwrapping
+        self.separatorByte = separator.asciiValue!   // swiftlint:disable:this force_unwrapping
     }
 }

Sources/Datadog/Core/Persistence/Reading/FileReader.swift

@@ -12,42 +12,91 @@ internal final class FileReader: Reader {
     private let dataFormat: DataFormat
     /// Orchestrator producing reference to readable file.
     private let orchestrator: FilesOrchestrator
+    private let encryption: DataEncryption?
     private let internalMonitor: InternalMonitor?
 
     /// Files marked as read.
-    private var filesRead: [ReadableFile] = []
+    private var filesRead: Set<String> = []
 
     init(
         dataFormat: DataFormat,
         orchestrator: FilesOrchestrator,
+        encryption: DataEncryption? = nil,
         internalMonitor: InternalMonitor? = nil
     ) {
         self.dataFormat = dataFormat
         self.orchestrator = orchestrator
+        self.encryption = encryption
         self.internalMonitor = internalMonitor
     }
 
     // MARK: - Reading batches
 
     func readNextBatch() -> Batch? {
-        if let file = orchestrator.getReadableFile(excludingFilesNamed: Set(filesRead.map { $0.name })) {
-            do {
-                let fileData = try file.read()
-                let batchData = dataFormat.prefixData + fileData + dataFormat.suffixData
-                return Batch(data: batchData, file: file)
-            } catch {
-                internalMonitor?.sdkLogger.error("Failed to read data from file", error: error)
-                return nil
-            }
+        guard let file = orchestrator.getReadableFile(excludingFilesNamed: filesRead) else {
+            return nil
+        }
+
+        do {
+            let fileData = try decrypt(data: file.read())
+            let batchData = dataFormat.prefixData + fileData + dataFormat.suffixData
+            return Batch(data: batchData, file: file)
+        } catch {
+            internalMonitor?.sdkLogger.error("Failed to read data from file", error: error)
+            return nil
+        }
+    }
+
+    /// Decrypts data if encryption is available.
+    ///
+    /// When encryption is provided, the data is splitted using data-format separator, each slices
+    /// is then decoded from base64 and decrypted. Data is finally re-joined with data-format separator.
+    ///
+    /// If no encryption, the data is returned.
+    ///
+    /// - Parameter data: The data to decrypt.
+    /// - Returns: Decrypted data.
+    private func decrypt(data: Data) -> Data {
+        guard let encryption = encryption else {
+            return data
         }
 
-        return nil
+        var failure: String? = nil
+        defer {
+            failure.map { userLogger.error($0) }
+        }
+
+        return data
+            // split data
+            .split(separator: dataFormat.separatorByte)
+            // decode base64 - report failure
+            .compactMap {
+                if let data = Data(base64Encoded: $0) {
+                    return data
+                }
+
+                failure = "🔥 Failed to decode base64 data before decryption"
+                return nil
+            }
+            // decrypt data - report failure
+            .compactMap { (data: Data) in
+                do {
+                    return try encryption.decrypt(data: data)
+                } catch {
+                    failure = "🔥 Failed to decrypt data with error: \(error)"
+                    return nil
+                }
+            }
+            // concat data
+            .reduce(Data()) { $0 + $1 + [dataFormat.separatorByte] }
+            // drop last separator
+            .dropLast()
     }
 
     // MARK: - Accepting batches
 
     func markBatchAsRead(_ batch: Batch) {
         orchestrator.delete(readableFile: batch.file)
-        filesRead.append(batch.file)
+        filesRead.insert(batch.file.name)
     }
 }

Sources/Datadog/Core/Persistence/Writing/FileWriter.swift

@@ -14,16 +14,19 @@ internal final class FileWriter: Writer {
     private let orchestrator: FilesOrchestrator
     /// JSON encoder used to encode data.
     private let jsonEncoder: JSONEncoder
+    private let encryption: DataEncryption?
     private let internalMonitor: InternalMonitor?
 
     init(
         dataFormat: DataFormat,
         orchestrator: FilesOrchestrator,
+        encryption: DataEncryption? = nil,
         internalMonitor: InternalMonitor? = nil
     ) {
         self.dataFormat = dataFormat
         self.orchestrator = orchestrator
-        self.jsonEncoder = JSONEncoder.default()
+        self.jsonEncoder = .default()
+        self.encryption = encryption
         self.internalMonitor = internalMonitor
     }
 
@@ -32,18 +35,34 @@ internal final class FileWriter: Writer {
     /// Encodes given value to JSON data and writes it to the file.
     func write<T: Encodable>(value: T) {
         do {
-            let data = try jsonEncoder.encode(value)
+            var data = try encode(value: value)
             let file = try orchestrator.getWritableFile(writeSize: UInt64(data.count))
 
-            if try file.size() == 0 {
-                try file.append(data: data)
-            } else {
-                let atomicData = dataFormat.separatorData + data
-                try file.append(data: atomicData)
+            if try file.size() > 0 {
+                data.insert(dataFormat.separatorByte, at: 0)
             }
+
+            try file.append(data: data)
         } catch {
             userLogger.error("🔥 Failed to write data: \(error)")
             internalMonitor?.sdkLogger.error("Failed to write data to file", error: error)
         }
     }
+
+    /// Encodes the given encodable value and encrypt it if encryption is available.
+    ///
+    /// If encryption is available, encryption result is base64 encoded.
+    ///
+    /// - Parameter value: The value to encode.
+    /// - Returns: Data representation of the value.
+    private func encode<T: Encodable>(value: T) throws -> Data {
+        let data = try jsonEncoder.encode(value)
+
+        guard let encryption = encryption else {
+            return data
+        }
+
+        return try encryption.encrypt(data: data)
+            .base64EncodedData()
+    }
 }

Sources/Datadog/Datadog.swift

@@ -227,7 +227,8 @@ public class Datadog {
             networkConnectionInfoProvider: networkConnectionInfoProvider,
             carrierInfoProvider: carrierInfoProvider,
             launchTimeProvider: launchTimeProvider,
-            appStateListener: AppStateListener(dateProvider: dateProvider)
+            appStateListener: AppStateListener(dateProvider: dateProvider),
+            encryption: configuration.common.encryption
         )
 
         if let internalMonitoringConfiguration = configuration.internalMonitoring {