Merge branch 'develop' into ganeshnj/feat/otel-tracer-tag

DataDog · Aug 21, 2024 · 3b26b85 · 3b26b85
2 parents 90b4538 + 475924e
commit 3b26b85
Show file tree

Hide file tree

Showing 35 changed files with 594 additions and 195 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 # Unreleased
 
 - [FIX] Propagate global Tracer tags to OpenTelemetry span attributes. See [#2000][]
+- [IMPROVEMENT] Send retry information with network requests (eg. retry_count, last_failure_status and idempotency key). See [#1991][]
 
 # 2.16.0 / 20-08-2024
 
@@ -748,6 +749,7 @@ Release `2.0` introduces breaking changes. Follow the [Migration Guide](MIGRATIO
 [#1973]: https://github.com/DataDog/dd-sdk-ios/pull/1973
 [#1988]: https://github.com/DataDog/dd-sdk-ios/pull/1988
 [#2000]: https://github.com/DataDog/dd-sdk-ios/pull/2000
+[#1991]: https://github.com/DataDog/dd-sdk-ios/pull/1991
 [@00fa9a]: https://github.com/00FA9A
 [@britton-earnin]: https://github.com/Britton-Earnin
 [@hengyu]: https://github.com/Hengyu

diff --git a/Datadog/Datadog.xcodeproj/project.pbxproj b/Datadog/Datadog.xcodeproj/project.pbxproj
@@ -112,6 +112,10 @@
 		3CCECDB02BC688120013C125 /* SpanIDGeneratorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CCECDAE2BC688120013C125 /* SpanIDGeneratorTests.swift */; };
 		3CCECDB22BC68A0A0013C125 /* SpanIDTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CCECDB12BC68A0A0013C125 /* SpanIDTests.swift */; };
 		3CCECDB32BC68A0A0013C125 /* SpanIDTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CCECDB12BC68A0A0013C125 /* SpanIDTests.swift */; };
+		3CD3A13A2C6C99ED00436A69 /* Data+Crypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C3C9E2B2C64F3CA003AF22F /* Data+Crypto.swift */; };
+		3CD3A13B2C6C99ED00436A69 /* Data+Crypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C3C9E2B2C64F3CA003AF22F /* Data+Crypto.swift */; };
+		3CD3A13C2C6C99FE00436A69 /* Data+CryptoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C3C9E2E2C64F470003AF22F /* Data+CryptoTests.swift */; };
+		3CD3A13D2C6C99FE00436A69 /* Data+CryptoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C3C9E2E2C64F470003AF22F /* Data+CryptoTests.swift */; };
 		3CDA3F7E2BCD866D005D2C13 /* DatadogSDKTesting in Frameworks */ = {isa = PBXBuildFile; productRef = 3CDA3F7D2BCD866D005D2C13 /* DatadogSDKTesting */; };
 		3CDA3F802BCD8687005D2C13 /* DatadogSDKTesting in Frameworks */ = {isa = PBXBuildFile; productRef = 3CDA3F7F2BCD8687005D2C13 /* DatadogSDKTesting */; };
 		3CE11A1129F7BE0900202522 /* DatadogWebViewTracking.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3CE119FE29F7BE0100202522 /* DatadogWebViewTracking.framework */; };
@@ -2112,6 +2116,8 @@
 		3C32359C2B55386C000B4258 /* OTelSpanLink.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OTelSpanLink.swift; sourceTree = "<group>"; };
 		3C32359F2B55387A000B4258 /* OTelSpanLinkTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OTelSpanLinkTests.swift; sourceTree = "<group>"; };
 		3C33E4062BEE35A7003B2988 /* RUMContextMocks.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RUMContextMocks.swift; sourceTree = "<group>"; };
+		3C3C9E2B2C64F3CA003AF22F /* Data+Crypto.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+Crypto.swift"; sourceTree = "<group>"; };
+		3C3C9E2E2C64F470003AF22F /* Data+CryptoTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+CryptoTests.swift"; sourceTree = "<group>"; };
 		3C3EF2AF2C1AEBAB009E9E57 /* LaunchReport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LaunchReport.swift; sourceTree = "<group>"; };
 		3C43A3862C188970000BFB21 /* WatchdogTerminationMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WatchdogTerminationMonitorTests.swift; sourceTree = "<group>"; };
 		3C4CF9972C47CC8C006DE1C0 /* MemoryWarningMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MemoryWarningMonitorTests.swift; sourceTree = "<group>"; };
@@ -5922,6 +5928,7 @@
 		D23039D6298D5235001A1FA3 /* Extensions */ = {
 			isa = PBXGroup;
 			children = (
+				3C3C9E2B2C64F3CA003AF22F /* Data+Crypto.swift */,
 				D23039D8298D5235001A1FA3 /* DatadogExtended.swift */,
 				D23354FB2A42E32000AFCAE2 /* InternalExtended.swift */,
 				D23039D7298D5235001A1FA3 /* Foundation+Datadog.swift */,
@@ -6134,6 +6141,7 @@
 		D263BCB129DB014900FA0E21 /* Extensions */ = {
 			isa = PBXGroup;
 			children = (
+				3C3C9E2E2C64F470003AF22F /* Data+CryptoTests.swift */,
 				D263BCB229DB014900FA0E21 /* FixedWidthInteger+ConvenienceTests.swift */,
 				D263BCB329DB014900FA0E21 /* TimeInterval+ConvenienceTests.swift */,
 			);
@@ -8677,6 +8685,7 @@
 				D23039E8298D5236001A1FA3 /* DatadogContext.swift in Sources */,
 				D23039FF298D5236001A1FA3 /* Foundation+Datadog.swift in Sources */,
 				D2F8235329915E12003C7E99 /* DatadogSite.swift in Sources */,
+				3CD3A13A2C6C99ED00436A69 /* Data+Crypto.swift in Sources */,
 				D2D3199A29E98D970004F169 /* DefaultJSONEncoder.swift in Sources */,
 				6128F56A2BA2237300D35B08 /* DataStore.swift in Sources */,
 				3C3EF2B02C1AEBAB009E9E57 /* LaunchReport.swift in Sources */,
@@ -9653,6 +9662,7 @@
 				D2DA2374298D57AA00C6C7E6 /* DatadogContext.swift in Sources */,
 				D2DA2375298D57AA00C6C7E6 /* Foundation+Datadog.swift in Sources */,
 				D2F8235429915E12003C7E99 /* DatadogSite.swift in Sources */,
+				3CD3A13B2C6C99ED00436A69 /* Data+Crypto.swift in Sources */,
 				D2D3199B29E98D970004F169 /* DefaultJSONEncoder.swift in Sources */,
 				6128F56B2BA2237300D35B08 /* DataStore.swift in Sources */,
 				3C3EF2B12C1AEBAB009E9E57 /* LaunchReport.swift in Sources */,
@@ -9731,6 +9741,7 @@
 				D2160CD429C0DF6700FAA9A5 /* NetworkInstrumentationFeatureTests.swift in Sources */,
 				D263BCB629DB014900FA0E21 /* TimeInterval+ConvenienceTests.swift in Sources */,
 				D2DA23AA298D58F400C6C7E6 /* FeatureMessageReceiverTests.swift in Sources */,
+				3CD3A13D2C6C99FE00436A69 /* Data+CryptoTests.swift in Sources */,
 				D2DA23A8298D58F400C6C7E6 /* DeviceInfoTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -9781,6 +9792,7 @@
 				D2160CD529C0DF6700FAA9A5 /* NetworkInstrumentationFeatureTests.swift in Sources */,
 				D263BCB729DB014900FA0E21 /* TimeInterval+ConvenienceTests.swift in Sources */,
 				D2DA23B8298D59DC00C6C7E6 /* FeatureMessageReceiverTests.swift in Sources */,
+				3CD3A13C2C6C99FE00436A69 /* Data+CryptoTests.swift in Sources */,
 				D2DA23BA298D59DC00C6C7E6 /* DeviceInfoTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

diff --git a/DatadogCore/Sources/Core/Upload/DataUploadStatus.swift b/DatadogCore/Sources/Core/Upload/DataUploadStatus.swift
@@ -72,28 +72,32 @@ internal struct DataUploadStatus {
     let userDebugDescription: String
 
     let error: DataUploadError?
+
+    let attempt: UInt
 }
 
 extension DataUploadStatus {
     // MARK: - Initialization
 
-    init(httpResponse: HTTPURLResponse, ddRequestID: String?) {
+    init(httpResponse: HTTPURLResponse, ddRequestID: String?, attempt: UInt) {
         let statusCode = HTTPResponseStatusCode(rawValue: httpResponse.statusCode) ?? .unexpected
 
         self.init(
             needsRetry: statusCode.needsRetry,
             responseCode: httpResponse.statusCode,
-            userDebugDescription: "[response code: \(httpResponse.statusCode) (\(statusCode)), request ID: \(ddRequestID ?? "(???)")]",
-            error: DataUploadError(status: httpResponse.statusCode)
+            userDebugDescription: "[response code: \(httpResponse.statusCode) (\(statusCode)), request ID: \(ddRequestID ?? "(???)")",
+            error: DataUploadError(status: httpResponse.statusCode),
+            attempt: attempt
         )
     }
 
-    init(networkError: Error) {
+    init(networkError: Error, attempt: UInt) {
         self.init(
             needsRetry: true, // retry this upload as it failed due to network transport isse
             responseCode: nil,
             userDebugDescription: "[error: \(DDError(error: networkError).message)]", // e.g. "[error: A data connection is not currently allowed]"
-            error: DataUploadError(networkError: networkError)
+            error: DataUploadError(networkError: networkError),
+            attempt: attempt
         )
     }
 }

diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift
@@ -44,6 +44,8 @@ internal class DataUploadWorker: DataUploadWorkerType {
     /// Background task coordinator responsible for registering and ending background tasks for UIKit targets.
     private var backgroundTaskCoordinator: BackgroundTaskCoordinator?
 
+    private var previousUploadStatus: DataUploadStatus?
+
     init(
         queue: DispatchQueue,
         fileReader: Reader,
@@ -113,8 +115,11 @@ internal class DataUploadWorker: DataUploadWorkerType {
                 do {
                     let uploadStatus = try self.dataUploader.upload(
                         events: batch.events,
-                        context: context
+                        context: context,
+                        previous: previousUploadStatus
                     )
+                    previousUploadStatus = uploadStatus
+
                     if uploadStatus.needsRetry {
                         DD.logger.debug("   → (\(self.featureName)) not delivered, will be retransmitted: \(uploadStatus.userDebugDescription)")
                         self.delay.increase()
@@ -129,6 +134,7 @@ internal class DataUploadWorker: DataUploadWorkerType {
                             batch,
                             reason: .intakeCode(responseCode: uploadStatus.responseCode)
                         )
+                        previousUploadStatus = nil
                     }
 
                     if let error = uploadStatus.error {
@@ -144,6 +150,7 @@ internal class DataUploadWorker: DataUploadWorkerType {
                 } catch let error {
                     // If upload can't be initiated do not retry, so drop the batch:
                     self.fileReader.markBatchAsRead(batch, reason: .invalid)
+                    previousUploadStatus = nil
                     self.telemetry.error("Failed to initiate '\(self.featureName)' data upload", error: error)
                 }
             }
@@ -173,12 +180,21 @@ internal class DataUploadWorker: DataUploadWorkerType {
                     // metrics or telemetry. This is legitimate as long as `flush()` routine is only available for testing
                     // purposes and never run in production apps.
                     self.fileReader.markBatchAsRead(nextBatch, reason: .flushed)
+                    previousUploadStatus = nil
                 }
                 do {
                     // Try uploading the batch and do one more retry on failure.
-                    _ = try self.dataUploader.upload(events: nextBatch.events, context: self.contextProvider.read())
+                    previousUploadStatus = try self.dataUploader.upload(
+                        events: nextBatch.events,
+                        context: self.contextProvider.read(),
+                        previous: previousUploadStatus
+                    )
                 } catch {
-                    _ = try? self.dataUploader.upload(events: nextBatch.events, context: self.contextProvider.read())
+                    previousUploadStatus = try? self.dataUploader.upload(
+                        events: nextBatch.events,
+                        context: self.contextProvider.read(),
+                        previous: previousUploadStatus
+                    )
                 }
             }
         }

diff --git a/DatadogCore/Sources/Core/Upload/DataUploader.swift b/DatadogCore/Sources/Core/Upload/DataUploader.swift
@@ -6,10 +6,11 @@
 
 import Foundation
 import DatadogInternal
+import CommonCrypto
 
 /// A type that performs data uploads.
 internal protocol DataUploaderType {
-    func upload(events: [Event], context: DatadogContext) throws -> DataUploadStatus
+    func upload(events: [Event], context: DatadogContext, previous: DataUploadStatus?) throws -> DataUploadStatus
 }
 
 /// Synchronously uploads data to server using `HTTPClient`.
@@ -19,7 +20,8 @@ internal final class DataUploader: DataUploaderType {
         needsRetry: false,
         responseCode: nil,
         userDebugDescription: "",
-        error: nil
+        error: nil,
+        attempt: 0
     )
 
     private let httpClient: HTTPClient
@@ -32,8 +34,17 @@ internal final class DataUploader: DataUploaderType {
 
     /// Uploads data synchronously (will block current thread) and returns the upload status.
     /// Uses timeout configured for `HTTPClient`.
-    func upload(events: [Event], context: DatadogContext) throws -> DataUploadStatus {
-        let request = try requestBuilder.request(for: events, with: context)
+    func upload(events: [Event], context: DatadogContext, previous: DataUploadStatus?) throws -> DataUploadStatus {
+        let attempt: UInt
+        if let previous = previous {
+            attempt = previous.attempt + 1
+        } else {
+            attempt = 0
+        }
+
+        let execution: ExecutionContext = .init(previousResponseCode: previous?.responseCode, attempt: attempt)
+        let request = try requestBuilder.request(for: events, with: context, execution: execution)
+
         let requestID = request.value(forHTTPHeaderField: URLRequestBuilder.HTTPHeader.ddRequestIDHeaderField)
 
         var uploadStatus: DataUploadStatus?
@@ -43,9 +54,16 @@ internal final class DataUploader: DataUploaderType {
         httpClient.send(request: request) { result in
             switch result {
             case .success(let httpResponse):
-                uploadStatus = DataUploadStatus(httpResponse: httpResponse, ddRequestID: requestID)
+                uploadStatus = DataUploadStatus(
+                    httpResponse: httpResponse,
+                    ddRequestID: requestID,
+                    attempt: attempt
+                )
             case .failure(let error):
-                uploadStatus = DataUploadStatus(networkError: error)
+                uploadStatus = DataUploadStatus(
+                    networkError: error,
+                    attempt: attempt
+                )
             }
 
             semaphore.signal()